closed-loop-default-detection 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. closed_loop_default_detection-0.1.0/LICENSE +21 -0
  2. closed_loop_default_detection-0.1.0/PKG-INFO +218 -0
  3. closed_loop_default_detection-0.1.0/README.md +184 -0
  4. closed_loop_default_detection-0.1.0/pyproject.toml +84 -0
  5. closed_loop_default_detection-0.1.0/setup.cfg +4 -0
  6. closed_loop_default_detection-0.1.0/src/cldd/__init__.py +96 -0
  7. closed_loop_default_detection-0.1.0/src/cldd/config.py +94 -0
  8. closed_loop_default_detection-0.1.0/src/cldd/correctors.py +172 -0
  9. closed_loop_default_detection-0.1.0/src/cldd/counterfactual.py +767 -0
  10. closed_loop_default_detection-0.1.0/src/cldd/diagnostics.py +107 -0
  11. closed_loop_default_detection-0.1.0/src/cldd/eval_default.py +161 -0
  12. closed_loop_default_detection-0.1.0/src/cldd/feedback.py +191 -0
  13. closed_loop_default_detection-0.1.0/src/cldd/fidelity.py +430 -0
  14. closed_loop_default_detection-0.1.0/src/cldd/loop.py +379 -0
  15. closed_loop_default_detection-0.1.0/src/cldd/model_pd.py +194 -0
  16. closed_loop_default_detection-0.1.0/src/cldd/py.typed +1 -0
  17. closed_loop_default_detection-0.1.0/src/cldd/reject_inference.py +264 -0
  18. closed_loop_default_detection-0.1.0/src/cldd/scm.py +1010 -0
  19. closed_loop_default_detection-0.1.0/src/cldd/synthetic.py +256 -0
  20. closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/PKG-INFO +218 -0
  21. closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/SOURCES.txt +37 -0
  22. closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/dependency_links.txt +1 -0
  23. closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/requires.txt +14 -0
  24. closed_loop_default_detection-0.1.0/src/closed_loop_default_detection.egg-info/top_level.txt +1 -0
  25. closed_loop_default_detection-0.1.0/tests/test_correctors.py +170 -0
  26. closed_loop_default_detection-0.1.0/tests/test_counterfactual.py +294 -0
  27. closed_loop_default_detection-0.1.0/tests/test_diagnostics.py +72 -0
  28. closed_loop_default_detection-0.1.0/tests/test_eval_default.py +89 -0
  29. closed_loop_default_detection-0.1.0/tests/test_exploration.py +90 -0
  30. closed_loop_default_detection-0.1.0/tests/test_feedback.py +118 -0
  31. closed_loop_default_detection-0.1.0/tests/test_fidelity.py +139 -0
  32. closed_loop_default_detection-0.1.0/tests/test_fidelity_report.py +107 -0
  33. closed_loop_default_detection-0.1.0/tests/test_loop.py +88 -0
  34. closed_loop_default_detection-0.1.0/tests/test_loop_scm.py +159 -0
  35. closed_loop_default_detection-0.1.0/tests/test_model_pd.py +98 -0
  36. closed_loop_default_detection-0.1.0/tests/test_reject_inference.py +161 -0
  37. closed_loop_default_detection-0.1.0/tests/test_scm_gating.py +150 -0
  38. closed_loop_default_detection-0.1.0/tests/test_sklearn_compat.py +191 -0
  39. closed_loop_default_detection-0.1.0/tests/test_synthetic.py +70 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hossain Pazooki
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,218 @@
1
+ Metadata-Version: 2.4
2
+ Name: closed-loop-default-detection
3
+ Version: 0.1.0
4
+ Summary: CLUE-style closed loop that measures selective-labels default detection on synthetic SMB lending cohorts and finds the PD model's operating frontier.
5
+ Author-email: Hossain Pazooki <hossain@pazooki.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/hossainpazooki/closed-loop-default-detection
8
+ Project-URL: Repository, https://github.com/hossainpazooki/closed-loop-default-detection
9
+ Project-URL: Issues, https://github.com/hossainpazooki/closed-loop-default-detection/issues
10
+ Keywords: selective-labels,probability-of-default,reject-inference,causal-inference,calibration,synthetic-data,off-policy-evaluation
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Financial and Insurance Industry
14
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Typing :: Typed
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: numpy>=2.0
22
+ Requires-Dist: pandas>=2.2
23
+ Requires-Dist: scikit-learn>=1.6
24
+ Requires-Dist: scipy>=1.11
25
+ Requires-Dist: matplotlib>=3.8
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=8.0; extra == "dev"
28
+ Requires-Dist: pytest-cov>=5; extra == "dev"
29
+ Provides-Extra: docs
30
+ Requires-Dist: sphinx>=7; extra == "docs"
31
+ Requires-Dist: furo; extra == "docs"
32
+ Requires-Dist: myst-parser>=2; extra == "docs"
33
+ Dynamic: license-file
34
+
35
+ # CLDD — closed-loop default detection
36
+
37
+ [![CI](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml/badge.svg)](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml)
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
39
+ [![Docs: Sphinx](https://img.shields.io/badge/docs-sphinx-blue.svg)](docs/)
40
+ [![Python ≥3.10](https://img.shields.io/badge/python-%E2%89%A53.10-blue.svg)](pyproject.toml)
41
+
42
+ **Stress-test a probability-of-default (PD) model under *selective labels* — and get the
43
+ severity at which it breaks.** Real lending data only labels the loans a prior underwriter
44
+ approved, so you cannot measure calibration on the applicants you *declined* — exactly where a
45
+ new model must still be right. CLDD builds synthetic lending worlds with planted ground truth,
46
+ hides labels the way real approval policies do, and grades every correction against that truth.
47
+
48
+ - **Deterministic** — byte-identical per seed, scikit-learn-only, no services or GPUs.
49
+ - **Pluggable** — correction levers (IPW, retrain, exploration, reject inference) are classes;
50
+ add yours by subclassing `Corrector`.
51
+ - **Honest by construction** — every number below recomputes from committed CSVs; limits are
52
+ reported, not smoothed over.
53
+
54
+ ## The result it produces
55
+
56
+ The loop escalates selection severity until correction fails and reports the **operating
57
+ frontier** — the last severity at which declined-cohort calibration still holds (target
58
+ ECE ≤ 0.10). From the committed runs (`artifacts/clue_frontier*.csv`, seed 42):
59
+
60
+ | Selection severity | 0.0 | 0.2 | 0.4 | 0.6 |
61
+ |---|---|---|---|---|
62
+ | Naive declined ECE (flat world) | 0.021 | 0.045 | 0.108 | 0.161 |
63
+ | **IPW-corrected** (flat world) | 0.020 | 0.038 | **0.086 ✓** | **0.154 ✗** |
64
+ | **IPW-corrected** (SCM world) | 0.036 | 0.038 | **0.097 ✓** | **0.244 ✗** |
65
+
66
+ Both worlds land the frontier at **severity 0.4**, and the counterfactual deliverable breaks at
67
+ the same boundary: across 25 seeds, g-computation cuts strong-propagation counterfactual MAE
68
+ from 0.099 to 0.086 (−13.5%, positive on 24/25 seeds, Wilcoxon p = 1.5e-7) *inside* the
69
+ frontier — and collapses to a negligible +0.0017 at full severity, where **no deployable
70
+ advantage is claimed**. One cause explains both: selection through an **unobserved
71
+ confounder**, which backdoor adjustment and IPW cannot fix. That single measured limit — not
72
+ an unverifiable score — is the deliverable.
73
+
74
+ Reproduce the headline from committed evidence: `python scripts/paired_significance.py`.
75
+ The full independent assessment (methodology, all numbers, what didn't hold) is the
76
+ accompanying article, [`FABLE.md`](FABLE.md).
77
+
78
+ ## Install
79
+
80
+ ```bash
81
+ pip install closed-loop-default-detection
82
+ ```
83
+
84
+ The import name is **`cldd`**. For development (tests, docs, the committed evidence),
85
+ install from source:
86
+
87
+ ```bash
88
+ git clone https://github.com/hossainpazooki/closed-loop-default-detection.git
89
+ cd closed-loop-default-detection
90
+ pip install -e ".[dev]"
91
+ ```
92
+
93
+ Python ≥ 3.10; dependencies are ranges (`numpy>=2.0`, `pandas>=2.2`, `scikit-learn>=1.6`,
94
+ `scipy>=1.11`, `matplotlib>=3.8`) so `cldd` sits alongside your stack. Exact pins for
95
+ float-exact reproduction: [`requirements-dev.txt`](requirements-dev.txt)
96
+ ([details](docs/validation.md)).
97
+
98
+ ## 60-second tour
99
+
100
+ ```python
101
+ from cldd import SelectiveLabelsLoop
102
+
103
+ result = SelectiveLabelsLoop(improve_mode="both").run() # "reweight" | "retrain" | "both"
104
+ print("Operating frontier:", result.frontier_severity)
105
+ for r in result.rounds:
106
+ print(r.selection_severity, r.naive.declined_ece, r.passed)
107
+ ```
108
+
109
+ ```mermaid
110
+ flowchart TD
111
+ A["<b>1. Generate</b><br/>synthetic cohort at a given selection severity<br/>plant true default, then hide it via the approval policy"]
112
+ B["<b>2. Measure</b><br/>train the PD model on approved rows only,<br/>score it against planted truth on the declined subpopulation"]
113
+ C["<b>3. Improve</b><br/>apply a correction lever:<br/>IPW reweight &middot; disjoint retrain &middot; exploration"]
114
+ D{"Corrected declined-cohort<br/>ECE &le; target?"}
115
+ E["<b>Operating frontier</b><br/>report the highest severity<br/>that still passes"]
116
+
117
+ A --> B --> C --> D
118
+ D -->|"yes &mdash; raise the severity"| A
119
+ D -->|"no &mdash; stop"| E
120
+ ```
121
+
122
+ A runnable end-to-end demo (classic + custom-lever paths) is
123
+ [`examples/quickstart.py`](examples/quickstart.py). Full mechanics, diagnostics, and the
124
+ feedback simulation: [docs/how-it-works.md](docs/how-it-works.md).
125
+
126
+ > **Scope.** CLDD is a synthetic **validation harness**, not a production pipeline: retraining
127
+ > and feedback are seeded simulations inside the harness; it never acts on live data or real
128
+ > lending decisions.
129
+
130
+ ## What's in the box
131
+
132
+ Everything is importable from top-level `cldd` (full reference: the [Sphinx docs](docs/)):
133
+
134
+ | Import | What it is |
135
+ |---|---|
136
+ | `SelectiveLabelsLoop` | the closed loop; `.run()` → `LoopResult` (frontier + per-round metrics) |
137
+ | `Corrector` + `NaiveCorrector`, `IPWReweightCorrector`, `DisjointRetrainCorrector`, `ExplorationCorrector` | the lever ABC and the four built-ins |
138
+ | `ReclassificationCorrector`, `AugmentationCorrector`, `FuzzyAugmentationCorrector`, `ParcellingCorrector` | four classic reject-inference methods, graded against planted truth ([honest results](docs/reject_inference.md)) |
139
+ | `SyntheticBorrowerGenerator`, `StructuralBorrowerGenerator` | the flat and fitted-SCM synthetic worlds |
140
+ | `run_counterfactual_eval`, `GComputationEstimator` | counterfactual validator (g-computation vs naive conditioning) |
141
+ | `FeedbackLoop` | model-in-the-loop selective-labels simulation |
142
+ | `positivity_diagnostics` | observable regime/drift alarm — needs **no** declined-row labels |
143
+ | `CalibratedPDClassifier` | the calibrated PD detector as a scikit-learn estimator |
144
+ | `cldd.fidelity.run_fidelity_gate` | SCM-vs-real **marginal**-fidelity gate (univariate marginals only) |
145
+
146
+ **Add a lever** by subclassing `Corrector` (`name`, `control_priority`, `apply`) and passing
147
+ `correctors=[NaiveCorrector(), MyCorrector()]` — the legacy `improve_mode` API is unchanged
148
+ and byte-identical. Contract details: [CONTRIBUTING.md](CONTRIBUTING.md).
149
+
150
+ **Use the detector from sklearn tooling** — `CalibratedPDClassifier` is a thin, tested wrapper
151
+ (binary-only; NaN features OK; the full `check_estimator` battery passes with zero failed
152
+ checks on scikit-learn 1.7.2–1.9.0; probabilities byte-identical to the research API):
153
+
154
+ ```python
155
+ from sklearn.model_selection import cross_val_score
156
+ from cldd import CalibratedPDClassifier
157
+
158
+ scores = cross_val_score(CalibratedPDClassifier(random_state=42), X, y, scoring="neg_brier_score")
159
+ ```
160
+
161
+ ## Command-line drivers
162
+
163
+ Each driver runs without install (adds `src/` to the path) and writes to `artifacts/`:
164
+
165
+ ```bash
166
+ python scripts/run_clue.py # the closed loop → frontier table + plot (--generator scm for the SCM world)
167
+ python scripts/run_seed_sweep.py --quick # counterfactual certification (drop --quick for all seeds)
168
+ python scripts/run_reject_inference.py # reject-inference levers vs the frontier
169
+ python scripts/run_exploration_sweep.py # frontier vs exploration budget
170
+ python scripts/run_feedback.py # model-in-the-loop feedback simulation
171
+ python scripts/paired_significance.py # recompute the headline stat from committed CSVs
172
+ ```
173
+
174
+ ## Validation
175
+
176
+ `pytest` — 123 tests, all synthetic, no real data needed. CI runs a pinned-repro job (exact
177
+ pins), a cross-version/OS compat matrix, and a strict docs build. Six float-sensitive tests
178
+ reproduce only under the pins in `requirements-dev.txt`; the optional marginal-fidelity gate
179
+ compares the SCM against a **private** real dataset via `CLDD_DATA_DIR` and is the only thing
180
+ that needs it. Details, reproducibility, and troubleshooting:
181
+ [docs/validation.md](docs/validation.md).
182
+
183
+ ## Documentation
184
+
185
+ | Where | What |
186
+ |---|---|
187
+ | [docs/quickstart.md](docs/quickstart.md) | run the loop, the counterfactual eval, the fidelity report |
188
+ | [docs/how-it-works.md](docs/how-it-works.md) | loop mechanics, diagnostics, feedback simulation, repo map |
189
+ | [docs/configuration.md](docs/configuration.md) | every knob (`config.py`) and the one env var |
190
+ | [docs/validation.md](docs/validation.md) | tests, gates, reproducibility, troubleshooting |
191
+ | [docs/reject_inference.md](docs/reject_inference.md) | the four RI methods and their honest (modest) results |
192
+ | [`FABLE.md`](FABLE.md) | **the accompanying article** — independent results & methodology assessment |
193
+
194
+ Build locally: `pip install -e ".[docs]" && sphinx-build -b html -W docs docs/_build/html`.
195
+
196
+ ## Status
197
+
198
+ `0.1.0` **alpha** on [PyPI](https://pypi.org/project/closed-loop-default-detection/),
199
+ changelog in [CHANGELOG.md](CHANGELOG.md). Shipped: the loop, both worlds, all levers, the
200
+ fidelity gate, the sklearn estimator, CI on three gates. CLDD began as a validation harness
201
+ for the Intuit TechWeek SMB Underwriting Challenge; it is not a submission and does not
202
+ alter challenge files.
203
+
204
+ ## Citation
205
+
206
+ Metadata in [`CITATION.cff`](CITATION.cff) (GitHub's "Cite this repository" reads it):
207
+
208
+ ```bibtex
209
+ @software{pazooki_cldd_2026,
210
+ author = {Pazooki, Hossain},
211
+ title = {{closed-loop-default-detection}: measuring selective-labels default
212
+ detection and the PD model's operating frontier},
213
+ year = {2026},
214
+ version = {0.1.0},
215
+ license = {MIT},
216
+ url = {https://github.com/hossainpazooki/closed-loop-default-detection}
217
+ }
218
+ ```
@@ -0,0 +1,184 @@
1
+ # CLDD — closed-loop default detection
2
+
3
+ [![CI](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml/badge.svg)](https://github.com/hossainpazooki/closed-loop-default-detection/actions/workflows/ci.yml)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
5
+ [![Docs: Sphinx](https://img.shields.io/badge/docs-sphinx-blue.svg)](docs/)
6
+ [![Python ≥3.10](https://img.shields.io/badge/python-%E2%89%A53.10-blue.svg)](pyproject.toml)
7
+
8
+ **Stress-test a probability-of-default (PD) model under *selective labels* — and get the
9
+ severity at which it breaks.** Real lending data only labels the loans a prior underwriter
10
+ approved, so you cannot measure calibration on the applicants you *declined* — exactly where a
11
+ new model must still be right. CLDD builds synthetic lending worlds with planted ground truth,
12
+ hides labels the way real approval policies do, and grades every correction against that truth.
13
+
14
+ - **Deterministic** — byte-identical per seed, scikit-learn-only, no services or GPUs.
15
+ - **Pluggable** — correction levers (IPW, retrain, exploration, reject inference) are classes;
16
+ add yours by subclassing `Corrector`.
17
+ - **Honest by construction** — every number below recomputes from committed CSVs; limits are
18
+ reported, not smoothed over.
19
+
20
+ ## The result it produces
21
+
22
+ The loop escalates selection severity until correction fails and reports the **operating
23
+ frontier** — the last severity at which declined-cohort calibration still holds (target
24
+ ECE ≤ 0.10). From the committed runs (`artifacts/clue_frontier*.csv`, seed 42):
25
+
26
+ | Selection severity | 0.0 | 0.2 | 0.4 | 0.6 |
27
+ |---|---|---|---|---|
28
+ | Naive declined ECE (flat world) | 0.021 | 0.045 | 0.108 | 0.161 |
29
+ | **IPW-corrected** (flat world) | 0.020 | 0.038 | **0.086 ✓** | **0.154 ✗** |
30
+ | **IPW-corrected** (SCM world) | 0.036 | 0.038 | **0.097 ✓** | **0.244 ✗** |
31
+
32
+ Both worlds land the frontier at **severity 0.4**, and the counterfactual deliverable breaks at
33
+ the same boundary: across 25 seeds, g-computation cuts strong-propagation counterfactual MAE
34
+ from 0.099 to 0.086 (−13.5%, positive on 24/25 seeds, Wilcoxon p = 1.5e-7) *inside* the
35
+ frontier — and collapses to a negligible +0.0017 at full severity, where **no deployable
36
+ advantage is claimed**. One cause explains both: selection through an **unobserved
37
+ confounder**, which backdoor adjustment and IPW cannot fix. That single measured limit — not
38
+ an unverifiable score — is the deliverable.
39
+
40
+ Reproduce the headline from committed evidence: `python scripts/paired_significance.py`.
41
+ The full independent assessment (methodology, all numbers, what didn't hold) is the
42
+ accompanying article, [`FABLE.md`](FABLE.md).
43
+
44
+ ## Install
45
+
46
+ ```bash
47
+ pip install closed-loop-default-detection
48
+ ```
49
+
50
+ The import name is **`cldd`**. For development (tests, docs, the committed evidence),
51
+ install from source:
52
+
53
+ ```bash
54
+ git clone https://github.com/hossainpazooki/closed-loop-default-detection.git
55
+ cd closed-loop-default-detection
56
+ pip install -e ".[dev]"
57
+ ```
58
+
59
+ Python ≥ 3.10; dependencies are ranges (`numpy>=2.0`, `pandas>=2.2`, `scikit-learn>=1.6`,
60
+ `scipy>=1.11`, `matplotlib>=3.8`) so `cldd` sits alongside your stack. Exact pins for
61
+ float-exact reproduction: [`requirements-dev.txt`](requirements-dev.txt)
62
+ ([details](docs/validation.md)).
63
+
64
+ ## 60-second tour
65
+
66
+ ```python
67
+ from cldd import SelectiveLabelsLoop
68
+
69
+ result = SelectiveLabelsLoop(improve_mode="both").run() # "reweight" | "retrain" | "both"
70
+ print("Operating frontier:", result.frontier_severity)
71
+ for r in result.rounds:
72
+ print(r.selection_severity, r.naive.declined_ece, r.passed)
73
+ ```
74
+
75
+ ```mermaid
76
+ flowchart TD
77
+ A["<b>1. Generate</b><br/>synthetic cohort at a given selection severity<br/>plant true default, then hide it via the approval policy"]
78
+ B["<b>2. Measure</b><br/>train the PD model on approved rows only,<br/>score it against planted truth on the declined subpopulation"]
79
+ C["<b>3. Improve</b><br/>apply a correction lever:<br/>IPW reweight &middot; disjoint retrain &middot; exploration"]
80
+ D{"Corrected declined-cohort<br/>ECE &le; target?"}
81
+ E["<b>Operating frontier</b><br/>report the highest severity<br/>that still passes"]
82
+
83
+ A --> B --> C --> D
84
+ D -->|"yes &mdash; raise the severity"| A
85
+ D -->|"no &mdash; stop"| E
86
+ ```
87
+
88
+ A runnable end-to-end demo (classic + custom-lever paths) is
89
+ [`examples/quickstart.py`](examples/quickstart.py). Full mechanics, diagnostics, and the
90
+ feedback simulation: [docs/how-it-works.md](docs/how-it-works.md).
91
+
92
+ > **Scope.** CLDD is a synthetic **validation harness**, not a production pipeline: retraining
93
+ > and feedback are seeded simulations inside the harness; it never acts on live data or real
94
+ > lending decisions.
95
+
96
+ ## What's in the box
97
+
98
+ Everything is importable from top-level `cldd` (full reference: the [Sphinx docs](docs/)):
99
+
100
+ | Import | What it is |
101
+ |---|---|
102
+ | `SelectiveLabelsLoop` | the closed loop; `.run()` → `LoopResult` (frontier + per-round metrics) |
103
+ | `Corrector` + `NaiveCorrector`, `IPWReweightCorrector`, `DisjointRetrainCorrector`, `ExplorationCorrector` | the lever ABC and the four built-ins |
104
+ | `ReclassificationCorrector`, `AugmentationCorrector`, `FuzzyAugmentationCorrector`, `ParcellingCorrector` | four classic reject-inference methods, graded against planted truth ([honest results](docs/reject_inference.md)) |
105
+ | `SyntheticBorrowerGenerator`, `StructuralBorrowerGenerator` | the flat and fitted-SCM synthetic worlds |
106
+ | `run_counterfactual_eval`, `GComputationEstimator` | counterfactual validator (g-computation vs naive conditioning) |
107
+ | `FeedbackLoop` | model-in-the-loop selective-labels simulation |
108
+ | `positivity_diagnostics` | observable regime/drift alarm — needs **no** declined-row labels |
109
+ | `CalibratedPDClassifier` | the calibrated PD detector as a scikit-learn estimator |
110
+ | `cldd.fidelity.run_fidelity_gate` | SCM-vs-real **marginal**-fidelity gate (univariate marginals only) |
111
+
112
+ **Add a lever** by subclassing `Corrector` (`name`, `control_priority`, `apply`) and passing
113
+ `correctors=[NaiveCorrector(), MyCorrector()]` — the legacy `improve_mode` API is unchanged
114
+ and byte-identical. Contract details: [CONTRIBUTING.md](CONTRIBUTING.md).
115
+
116
+ **Use the detector from sklearn tooling** — `CalibratedPDClassifier` is a thin, tested wrapper
117
+ (binary-only; NaN features OK; the full `check_estimator` battery passes with zero failed
118
+ checks on scikit-learn 1.7.2–1.9.0; probabilities byte-identical to the research API):
119
+
120
+ ```python
121
+ from sklearn.model_selection import cross_val_score
122
+ from cldd import CalibratedPDClassifier
123
+
124
+ scores = cross_val_score(CalibratedPDClassifier(random_state=42), X, y, scoring="neg_brier_score")
125
+ ```
126
+
127
+ ## Command-line drivers
128
+
129
+ Each driver runs without install (adds `src/` to the path) and writes to `artifacts/`:
130
+
131
+ ```bash
132
+ python scripts/run_clue.py # the closed loop → frontier table + plot (--generator scm for the SCM world)
133
+ python scripts/run_seed_sweep.py --quick # counterfactual certification (drop --quick for all seeds)
134
+ python scripts/run_reject_inference.py # reject-inference levers vs the frontier
135
+ python scripts/run_exploration_sweep.py # frontier vs exploration budget
136
+ python scripts/run_feedback.py # model-in-the-loop feedback simulation
137
+ python scripts/paired_significance.py # recompute the headline stat from committed CSVs
138
+ ```
139
+
140
+ ## Validation
141
+
142
+ `pytest` — 123 tests, all synthetic, no real data needed. CI runs a pinned-repro job (exact
143
+ pins), a cross-version/OS compat matrix, and a strict docs build. Six float-sensitive tests
144
+ reproduce only under the pins in `requirements-dev.txt`; the optional marginal-fidelity gate
145
+ compares the SCM against a **private** real dataset via `CLDD_DATA_DIR` and is the only thing
146
+ that needs it. Details, reproducibility, and troubleshooting:
147
+ [docs/validation.md](docs/validation.md).
148
+
149
+ ## Documentation
150
+
151
+ | Where | What |
152
+ |---|---|
153
+ | [docs/quickstart.md](docs/quickstart.md) | run the loop, the counterfactual eval, the fidelity report |
154
+ | [docs/how-it-works.md](docs/how-it-works.md) | loop mechanics, diagnostics, feedback simulation, repo map |
155
+ | [docs/configuration.md](docs/configuration.md) | every knob (`config.py`) and the one env var |
156
+ | [docs/validation.md](docs/validation.md) | tests, gates, reproducibility, troubleshooting |
157
+ | [docs/reject_inference.md](docs/reject_inference.md) | the four RI methods and their honest (modest) results |
158
+ | [`FABLE.md`](FABLE.md) | **the accompanying article** — independent results & methodology assessment |
159
+
160
+ Build locally: `pip install -e ".[docs]" && sphinx-build -b html -W docs docs/_build/html`.
161
+
162
+ ## Status
163
+
164
+ `0.1.0` **alpha** on [PyPI](https://pypi.org/project/closed-loop-default-detection/),
165
+ changelog in [CHANGELOG.md](CHANGELOG.md). Shipped: the loop, both worlds, all levers, the
166
+ fidelity gate, the sklearn estimator, CI on three gates. CLDD began as a validation harness
167
+ for the Intuit TechWeek SMB Underwriting Challenge; it is not a submission and does not
168
+ alter challenge files.
169
+
170
+ ## Citation
171
+
172
+ Metadata in [`CITATION.cff`](CITATION.cff) (GitHub's "Cite this repository" reads it):
173
+
174
+ ```bibtex
175
+ @software{pazooki_cldd_2026,
176
+ author = {Pazooki, Hossain},
177
+ title = {{closed-loop-default-detection}: measuring selective-labels default
178
+ detection and the PD model's operating frontier},
179
+ year = {2026},
180
+ version = {0.1.0},
181
+ license = {MIT},
182
+ url = {https://github.com/hossainpazooki/closed-loop-default-detection}
183
+ }
184
+ ```
@@ -0,0 +1,84 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "closed-loop-default-detection"
7
+ version = "0.1.0"
8
+ description = "CLUE-style closed loop that measures selective-labels default detection on synthetic SMB lending cohorts and finds the PD model's operating frontier."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Hossain Pazooki", email = "hossain@pazooki.com" }]
14
+ keywords = [
15
+ "selective-labels",
16
+ "probability-of-default",
17
+ "reject-inference",
18
+ "causal-inference",
19
+ "calibration",
20
+ "synthetic-data",
21
+ "off-policy-evaluation",
22
+ ]
23
+ classifiers = [
24
+ "Development Status :: 3 - Alpha",
25
+ "Intended Audience :: Science/Research",
26
+ "Intended Audience :: Financial and Insurance Industry",
27
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
28
+ "Programming Language :: Python :: 3",
29
+ "Operating System :: OS Independent",
30
+ "Typing :: Typed",
31
+ ]
32
+ # Library dependencies are declared as ranges so the package installs alongside
33
+ # a practitioner's own stack. The EXACT provenance pins under which the committed
34
+ # artifacts and the frozen byte-identity baseline were captured
35
+ # (scikit-learn 1.9.0 / numpy 2.4.6, Python 3.14.2) live in requirements-dev.txt
36
+ # and are exercised by the CI `pinned-repro` job. HistGradientBoosting float
37
+ # output shifts across scikit-learn releases, so a handful of float-exact tests
38
+ # (marked `pinned`) are reproducible only under those pins.
39
+ dependencies = [
40
+ "numpy>=2.0",
41
+ "pandas>=2.2",
42
+ "scikit-learn>=1.6",
43
+ "scipy>=1.11",
44
+ "matplotlib>=3.8",
45
+ ]
46
+
47
+ [project.optional-dependencies]
48
+ dev = ["pytest>=8.0", "pytest-cov>=5"]
49
+ docs = ["sphinx>=7", "furo", "myst-parser>=2"]
50
+
51
+ [project.urls]
52
+ Homepage = "https://github.com/hossainpazooki/closed-loop-default-detection"
53
+ Repository = "https://github.com/hossainpazooki/closed-loop-default-detection"
54
+ Issues = "https://github.com/hossainpazooki/closed-loop-default-detection/issues"
55
+
56
+ [tool.setuptools.packages.find]
57
+ where = ["src"]
58
+
59
+ # Ship the PEP 561 typing marker so downstream type-checkers honor cldd's hints.
60
+ [tool.setuptools.package-data]
61
+ cldd = ["py.typed"]
62
+
63
+ [tool.pytest.ini_options]
64
+ testpaths = ["tests"]
65
+ addopts = "-q"
66
+ markers = [
67
+ "pinned: asserts exact or tight floating-point output of the calibrated PD model; reproducible only under the pinned scikit-learn/numpy in requirements-dev.txt. The CI compat matrix deselects these with `-m 'not pinned'`.",
68
+ ]
69
+
70
+ # Coverage is opt-in (run `pytest --cov=cldd`); it is not wired into addopts so a
71
+ # plain `pytest` stays lightweight. The private-data fidelity path (cldd/fidelity.py)
72
+ # is only exercised when the real dataset is present, so its data-loading branch is
73
+ # expected to be uncovered on public CI by design — see README.
74
+ [tool.coverage.run]
75
+ source = ["cldd"]
76
+ branch = true
77
+
78
+ [tool.coverage.report]
79
+ show_missing = true
80
+ exclude_also = [
81
+ "if TYPE_CHECKING:",
82
+ "raise NotImplementedError",
83
+ "if __name__ == .__main__.:",
84
+ ]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,96 @@
1
+ """Closed-loop default-rate detection — a CLUE-style harness for selective labels.
2
+
3
+ generate (``synthetic``) -> measure (``eval_default``) -> improve/frontier
4
+ (``loop``). See README.md for the CLUE mapping and how to run.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from importlib.metadata import PackageNotFoundError, version as _pkg_version
10
+
11
+ try: # pyproject `version` is the single source of truth (read from installed metadata)
12
+ __version__ = _pkg_version("closed-loop-default-detection")
13
+ except PackageNotFoundError: # running from a source tree with no install
14
+ __version__ = "0.1.0"
15
+
16
+ from .correctors import (
17
+ CorrectionOutcome,
18
+ Corrector,
19
+ CorrectorContext,
20
+ DisjointRetrainCorrector,
21
+ ExplorationCorrector,
22
+ IPWReweightCorrector,
23
+ NaiveCorrector,
24
+ )
25
+ from .counterfactual import (
26
+ CounterfactualResult,
27
+ GComputationEstimator,
28
+ generate_queries,
29
+ run_counterfactual_eval,
30
+ )
31
+ from .diagnostics import PositivityDiagnostics, positivity_diagnostics
32
+ from .reject_inference import (
33
+ AugmentationCorrector,
34
+ FuzzyAugmentationCorrector,
35
+ ParcellingCorrector,
36
+ ReclassificationCorrector,
37
+ RejectInferenceCorrector,
38
+ )
39
+ from .eval_default import PdDetectionResult, fit_observed_model, score_pd_detection
40
+ from .model_pd import CalibratedPDClassifier, CalibratedPDModel
41
+ from .feedback import FeedbackLoop, FeedbackResult, GenerationResult
42
+ from .loop import LeverMetrics, LoopResult, RoundResult, SelectiveLabelsLoop
43
+ from .scm import (
44
+ BANK_FEED_COLUMNS,
45
+ FEATURE_COLUMNS,
46
+ INTERVENABLE_FEATURES,
47
+ InterventionResult,
48
+ SCMState,
49
+ StructuralBorrowerGenerator,
50
+ dag_children,
51
+ dag_parents,
52
+ )
53
+ from .synthetic import SyntheticBorrowerGenerator
54
+
55
+ __all__ = [
56
+ "__version__",
57
+ "SyntheticBorrowerGenerator",
58
+ "StructuralBorrowerGenerator",
59
+ "SCMState",
60
+ "InterventionResult",
61
+ "INTERVENABLE_FEATURES",
62
+ "FEATURE_COLUMNS",
63
+ "BANK_FEED_COLUMNS",
64
+ "dag_children",
65
+ "dag_parents",
66
+ "SelectiveLabelsLoop",
67
+ "LoopResult",
68
+ "RoundResult",
69
+ "LeverMetrics",
70
+ "Corrector",
71
+ "CorrectorContext",
72
+ "CorrectionOutcome",
73
+ "NaiveCorrector",
74
+ "IPWReweightCorrector",
75
+ "DisjointRetrainCorrector",
76
+ "ExplorationCorrector",
77
+ "RejectInferenceCorrector",
78
+ "ReclassificationCorrector",
79
+ "AugmentationCorrector",
80
+ "FuzzyAugmentationCorrector",
81
+ "ParcellingCorrector",
82
+ "FeedbackLoop",
83
+ "FeedbackResult",
84
+ "GenerationResult",
85
+ "PositivityDiagnostics",
86
+ "positivity_diagnostics",
87
+ "PdDetectionResult",
88
+ "fit_observed_model",
89
+ "score_pd_detection",
90
+ "CalibratedPDModel",
91
+ "CalibratedPDClassifier",
92
+ "CounterfactualResult",
93
+ "GComputationEstimator",
94
+ "run_counterfactual_eval",
95
+ "generate_queries",
96
+ ]