closed-loop-default-detection 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cldd/__init__.py +96 -0
- cldd/config.py +94 -0
- cldd/correctors.py +172 -0
- cldd/counterfactual.py +767 -0
- cldd/diagnostics.py +107 -0
- cldd/eval_default.py +161 -0
- cldd/feedback.py +191 -0
- cldd/fidelity.py +430 -0
- cldd/loop.py +379 -0
- cldd/model_pd.py +194 -0
- cldd/py.typed +1 -0
- cldd/reject_inference.py +264 -0
- cldd/scm.py +1010 -0
- cldd/synthetic.py +256 -0
- closed_loop_default_detection-0.1.0.dist-info/METADATA +218 -0
- closed_loop_default_detection-0.1.0.dist-info/RECORD +19 -0
- closed_loop_default_detection-0.1.0.dist-info/WHEEL +5 -0
- closed_loop_default_detection-0.1.0.dist-info/licenses/LICENSE +21 -0
- closed_loop_default_detection-0.1.0.dist-info/top_level.txt +1 -0
cldd/scm.py
ADDED
|
@@ -0,0 +1,1010 @@
|
|
|
1
|
+
"""Structural causal model for SMB borrowers with KNOWN-TRUTH counterfactuals.
|
|
2
|
+
|
|
3
|
+
This is the *causal* upgrade of :mod:`cldd.synthetic`. Where ``synthetic.py``
|
|
4
|
+
draws features independently and plants a logistic ``true_default``, this module
|
|
5
|
+
implements a **layered structural causal model (SCM)** fitted to the real Intuit
|
|
6
|
+
SMB underwriting data, so that the *true* post-intervention default probability of
|
|
7
|
+
``do(feature = value)`` is computable by propagating the change through the DAG.
|
|
8
|
+
|
|
9
|
+
The contract with :class:`cldd.loop.SelectiveLabelsLoop` is preserved: a cohort
|
|
10
|
+
dict with keys ``features, true_default, approved, prior_score, ground_truth`` —
|
|
11
|
+
so this generator is drop-in for the loop. It adds the counterfactual machinery
|
|
12
|
+
(``true_pd``, ``do_intervention``) plus the survival/observation fields.
|
|
13
|
+
|
|
14
|
+
Design — how known-truth interventions stay exact
|
|
15
|
+
--------------------------------------------------
|
|
16
|
+
Every node is generated as ``value = F^{-1}(Phi(z))`` where ``z`` is a per-unit
|
|
17
|
+
**latent Gaussian** that is a deterministic linear function of its parents'
|
|
18
|
+
latents plus a frozen exogenous noise term, and ``F^{-1}`` is the fitted marginal
|
|
19
|
+
quantile (Gaussian-copula construction). The exogenous noise draws are frozen on
|
|
20
|
+
``generate_cohort`` and stored on the returned :class:`SCMState`. To compute
|
|
21
|
+
``do(X = v)`` we:
|
|
22
|
+
|
|
23
|
+
1. clamp node ``X``'s realized value to ``v`` (and invert it to a clamped latent
|
|
24
|
+
``z_X`` so X's *descendants* see the intervention),
|
|
25
|
+
2. re-propagate every descendant of ``X`` through the same structural equations
|
|
26
|
+
with the SAME frozen noise (ancestors and exogenous noise unchanged),
|
|
27
|
+
3. recompute the latent default logit and return ``sigmoid(logit)``.
|
|
28
|
+
|
|
29
|
+
Because nothing but ``X``'s descendants change and the noise is identical,
|
|
30
|
+
``do(X = observed_value)`` reproduces the baseline ``true_pd`` exactly (no-op
|
|
31
|
+
invariance). The bank-feed block is a structural switch: toggling
|
|
32
|
+
``has_linked_bank_feed`` True regenerates the whole 6-node block from the SCM;
|
|
33
|
+
toggling it False deletes it (sets the block to NaN).
|
|
34
|
+
|
|
35
|
+
All randomness flows through one seeded ``numpy.random.Generator(PCG64(seed))``
|
|
36
|
+
for byte-identical reproducibility per seed.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
|
|
43
|
+
import numpy as np
|
|
44
|
+
import pandas as pd
|
|
45
|
+
from scipy import stats
|
|
46
|
+
|
|
47
|
+
from . import config
|
|
48
|
+
|
|
49
|
+
# --------------------------------------------------------------------------- #
|
|
50
|
+
# Feature columns (superset of synthetic.FEATURE_COLUMNS; covers all 16
|
|
51
|
+
# intervenable features + business_identity roots + prior_underwriter group).
|
|
52
|
+
# --------------------------------------------------------------------------- #
|
|
53
|
+
|
|
54
|
+
#: Observed model matrix exposed to the PD model and the loop. Ordered roots ->
|
|
55
|
+
#: self-reported -> bank-feed (gated) -> bureau -> platform -> application ->
|
|
56
|
+
#: prior-underwriter. Bank-feed columns are NaN when no feed is linked.
|
|
57
|
+
FEATURE_COLUMNS = [
|
|
58
|
+
# business_identity roots
|
|
59
|
+
"sector",
|
|
60
|
+
"geography_region",
|
|
61
|
+
"vintage_years",
|
|
62
|
+
"employee_count_bucket",
|
|
63
|
+
# self_reported (intervenable: stated_annual_revenue, stated_time_in_business, requested_amount)
|
|
64
|
+
"stated_annual_revenue",
|
|
65
|
+
"stated_time_in_business",
|
|
66
|
+
"requested_amount",
|
|
67
|
+
# bank_feed GATED block (all intervenable)
|
|
68
|
+
"has_linked_bank_feed",
|
|
69
|
+
"observed_monthly_revenue_avg_3mo",
|
|
70
|
+
"observed_revenue_trend_3mo",
|
|
71
|
+
"observed_revenue_volatility",
|
|
72
|
+
"observed_cash_balance_p10",
|
|
73
|
+
"observed_overdraft_count_3mo",
|
|
74
|
+
"payroll_regularity_score",
|
|
75
|
+
# bureau_credit (intervenable: aggregate_credit_utilization, recent_inquiries_count_6mo,
|
|
76
|
+
# existing_debt_obligations, owner_personal_credit_band)
|
|
77
|
+
"aggregate_credit_utilization",
|
|
78
|
+
"recent_inquiries_count_6mo",
|
|
79
|
+
"existing_debt_obligations",
|
|
80
|
+
"owner_personal_credit_band",
|
|
81
|
+
# platform_engagement (intervenable: invoice_payment_delinquency_rate)
|
|
82
|
+
"invoice_payment_delinquency_rate",
|
|
83
|
+
"prior_loans_count",
|
|
84
|
+
"prior_loans_default_count",
|
|
85
|
+
# application_context (intervenable: application_channel, multi_lender_inquiry_count_30d)
|
|
86
|
+
"application_channel",
|
|
87
|
+
"multi_lender_inquiry_count_30d",
|
|
88
|
+
"requested_amount_to_observed_revenue", # derived
|
|
89
|
+
# prior_underwriter group
|
|
90
|
+
"prior_underwriter_score",
|
|
91
|
+
"prior_decision",
|
|
92
|
+
"prior_approved_amount",
|
|
93
|
+
# derived leverage ratio (exposed; used by the risk logit)
|
|
94
|
+
"debt_to_revenue",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
#: The 16 intervenable=True features (intervention engine validates against this).
|
|
98
|
+
INTERVENABLE_FEATURES = (
|
|
99
|
+
"stated_annual_revenue",
|
|
100
|
+
"stated_time_in_business",
|
|
101
|
+
"requested_amount",
|
|
102
|
+
"observed_monthly_revenue_avg_3mo",
|
|
103
|
+
"observed_revenue_trend_3mo",
|
|
104
|
+
"observed_revenue_volatility",
|
|
105
|
+
"observed_cash_balance_p10",
|
|
106
|
+
"observed_overdraft_count_3mo",
|
|
107
|
+
"payroll_regularity_score",
|
|
108
|
+
"aggregate_credit_utilization",
|
|
109
|
+
"recent_inquiries_count_6mo",
|
|
110
|
+
"existing_debt_obligations",
|
|
111
|
+
"owner_personal_credit_band",
|
|
112
|
+
"invoice_payment_delinquency_rate",
|
|
113
|
+
"application_channel",
|
|
114
|
+
"multi_lender_inquiry_count_30d",
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
#: Columns null exactly when ``has_linked_bank_feed`` is False (structural switch).
|
|
118
|
+
BANK_FEED_COLUMNS = (
|
|
119
|
+
"observed_monthly_revenue_avg_3mo",
|
|
120
|
+
"observed_revenue_trend_3mo",
|
|
121
|
+
"observed_revenue_volatility",
|
|
122
|
+
"observed_cash_balance_p10",
|
|
123
|
+
"observed_overdraft_count_3mo",
|
|
124
|
+
"payroll_regularity_score",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
#: Train [min, max] support per feature, for the in-support counterfactual check.
|
|
128
|
+
#: For integer/count features the check compares to [min, max] (not p99) so p99
|
|
129
|
+
#: integer-tie ceilings are not flagged as extrapolation (per the query spec).
|
|
130
|
+
FEATURE_SUPPORT = {
|
|
131
|
+
"stated_annual_revenue": (156_537.0, 43_500_000.0),
|
|
132
|
+
"stated_time_in_business": (0.033, 40.428),
|
|
133
|
+
"requested_amount": (7_590.0, 45_903.0),
|
|
134
|
+
"observed_monthly_revenue_avg_3mo": (14_996.0, 2_080_000.0),
|
|
135
|
+
"observed_revenue_trend_3mo": (-2.127, 2.799),
|
|
136
|
+
"observed_revenue_volatility": (0.088, 8.157),
|
|
137
|
+
"observed_cash_balance_p10": (-12_373.0, 13_611.0),
|
|
138
|
+
"observed_overdraft_count_3mo": (0.0, 6.0),
|
|
139
|
+
"payroll_regularity_score": (0.0, 1.0),
|
|
140
|
+
"aggregate_credit_utilization": (0.002, 0.993),
|
|
141
|
+
"recent_inquiries_count_6mo": (0.0, 6.0),
|
|
142
|
+
"existing_debt_obligations": (970.0, 69_239.0),
|
|
143
|
+
"owner_personal_credit_band": (0.0, 4.0),
|
|
144
|
+
"invoice_payment_delinquency_rate": (0.003, 0.912),
|
|
145
|
+
"application_channel": (0.0, 2.0),
|
|
146
|
+
"multi_lender_inquiry_count_30d": (0.0, 5.0),
|
|
147
|
+
"vintage_years": (0.0, 60.0),
|
|
148
|
+
"existing_debt_obligations_": (970.0, 69_239.0),
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
_NUMERIC_NOOP_TOL = lambda v: 1e-6 + 1e-3 * abs(float(v)) # noqa: E731 (query-spec tolerance)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --------------------------------------------------------------------------- #
|
|
155
|
+
# Fitted marginal quantile transforms: u in (0,1) -> feature value.
|
|
156
|
+
# Each verified vs real median/p99 in the marginals spec.
|
|
157
|
+
# --------------------------------------------------------------------------- #
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _q_lognormal(u, mu, sigma):
|
|
161
|
+
return np.exp(mu + sigma * stats.norm.ppf(u))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _q_normal(u, mu, sigma, lo=None, hi=None):
|
|
165
|
+
x = mu + sigma * stats.norm.ppf(u)
|
|
166
|
+
if lo is not None:
|
|
167
|
+
x = np.maximum(x, lo)
|
|
168
|
+
if hi is not None:
|
|
169
|
+
x = np.minimum(x, hi)
|
|
170
|
+
return x
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _q_gamma(u, a, scale, lo=None):
|
|
174
|
+
x = stats.gamma.ppf(u, a, scale=scale)
|
|
175
|
+
if lo is not None:
|
|
176
|
+
x = np.maximum(x, lo)
|
|
177
|
+
return x
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _q_beta(u, a, b):
|
|
181
|
+
return stats.beta.ppf(u, a, b)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _q_poisson(u, lam):
|
|
185
|
+
return stats.poisson.ppf(np.clip(u, 1e-9, 1 - 1e-9), lam).astype(float)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _q_categorical(u, probs):
|
|
189
|
+
"""Inverse-CDF for an integer-coded categorical with level probs ``probs``."""
|
|
190
|
+
edges = np.cumsum(np.asarray(probs, dtype=float))
|
|
191
|
+
edges[-1] = 1.0 + 1e-9
|
|
192
|
+
return np.searchsorted(edges, np.asarray(u), side="right").astype(float)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _sigmoid(x):
|
|
196
|
+
return 1.0 / (1.0 + np.exp(-x))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _solve_intercept(logit_no_intercept: np.ndarray, target_rate: float) -> float:
|
|
200
|
+
"""Bisection for the intercept ``c`` s.t. ``mean(sigmoid(logit + c)) == target``."""
|
|
201
|
+
lo, hi = -12.0, 12.0
|
|
202
|
+
for _ in range(80):
|
|
203
|
+
mid = 0.5 * (lo + hi)
|
|
204
|
+
rate = float(np.mean(_sigmoid(logit_no_intercept + mid)))
|
|
205
|
+
if rate < target_rate:
|
|
206
|
+
lo = mid
|
|
207
|
+
else:
|
|
208
|
+
hi = mid
|
|
209
|
+
return 0.5 * (lo + hi)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _zfix(x, mean, std):
|
|
213
|
+
"""Standardize with FIXED (cohort-baseline) mean/std so the risk logit is a
|
|
214
|
+
fixed structural function — interventions move the input, not the scaler."""
|
|
215
|
+
std = std if (np.isfinite(std) and std > 0) else 1.0
|
|
216
|
+
z = (np.asarray(x, dtype=float) - mean) / std
|
|
217
|
+
return np.nan_to_num(z, nan=0.0)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# --------------------------------------------------------------------------- #
|
|
221
|
+
# SCM state returned to callers (counterfactual engine reads this).
|
|
222
|
+
# --------------------------------------------------------------------------- #
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
@dataclass
|
|
226
|
+
class SCMState:
|
|
227
|
+
"""Frozen per-cohort SCM state the counterfactual engine needs.
|
|
228
|
+
|
|
229
|
+
``noise`` holds the per-unit exogenous uniform draws for every node; ``latent``
|
|
230
|
+
holds the realized latent Gaussians; ``values`` holds the realized feature
|
|
231
|
+
arrays (pre-gating, i.e. bank-feed block populated for all units). ``has_feed``
|
|
232
|
+
is the structural switch. ``risk_scaler`` are the FIXED standardization
|
|
233
|
+
params for the default logit so it is a stable structural function.
|
|
234
|
+
"""
|
|
235
|
+
|
|
236
|
+
n: int
|
|
237
|
+
seed: int
|
|
238
|
+
noise: dict = field(default_factory=dict) # node -> per-unit U(0,1) ndarray
|
|
239
|
+
latent: dict = field(default_factory=dict) # node -> per-unit latent Gaussian
|
|
240
|
+
values: dict = field(default_factory=dict) # node -> per-unit realized value (ungated)
|
|
241
|
+
has_feed: np.ndarray = field(default=None) # bool ndarray
|
|
242
|
+
risk_scaler: dict = field(default_factory=dict) # term name -> (mean, std)
|
|
243
|
+
risk_intercept: float = 0.0
|
|
244
|
+
confounder_u: np.ndarray = field(default=None) # unobserved confounder (frozen)
|
|
245
|
+
generator: "StructuralBorrowerGenerator" = None # back-reference for do_intervention
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@dataclass
|
|
249
|
+
class InterventionResult:
|
|
250
|
+
"""Return of :meth:`StructuralBorrowerGenerator.do_intervention`."""
|
|
251
|
+
|
|
252
|
+
feature: str
|
|
253
|
+
value: float
|
|
254
|
+
true_pd: np.ndarray # post-intervention true PD, shape (n,)
|
|
255
|
+
baseline_pd: np.ndarray # pre-intervention true PD, shape (n,)
|
|
256
|
+
effect: np.ndarray # true_pd - baseline_pd, shape (n,)
|
|
257
|
+
intervenable: bool # False => engine refuses (non-intervenable feature)
|
|
258
|
+
in_support: bool # value within train [min, max]
|
|
259
|
+
is_noop: bool # value ~= observed (per-unit baseline preserved)
|
|
260
|
+
structural: bool = False # True => structural (information) switch, e.g.
|
|
261
|
+
# has_linked_bank_feed: manipulable but NOT one
|
|
262
|
+
# of the 16 policy-intervenable features.
|
|
263
|
+
note: str = ""
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class StructuralBorrowerGenerator:
|
|
267
|
+
"""Layered, fitted SCM for SMB borrowers with computable counterfactuals.
|
|
268
|
+
|
|
269
|
+
Drop-in for :class:`cldd.synthetic.SyntheticBorrowerGenerator` (same cohort
|
|
270
|
+
contract) plus ``true_pd`` / ``do_intervention`` for known-truth interventions.
|
|
271
|
+
"""
|
|
272
|
+
|
|
273
|
+
def __init__(
|
|
274
|
+
self,
|
|
275
|
+
n_applicants: int = config.DEFAULT_N_APPLICANTS,
|
|
276
|
+
selection_severity: float = 1.0,
|
|
277
|
+
approval_rate: float = config.DEFAULT_APPROVAL_RATE,
|
|
278
|
+
target_base_rate: float = config.TARGET_BASE_DEFAULT_RATE,
|
|
279
|
+
bank_feed_rate: float = 0.643157,
|
|
280
|
+
unobserved_strength: float = 0.55,
|
|
281
|
+
seed: int = config.RANDOM_SEED,
|
|
282
|
+
independent_selection_noise: bool = False,
|
|
283
|
+
) -> None:
|
|
284
|
+
# independent_selection_noise: by default the (1 - severity) blend noise in
|
|
285
|
+
# ``_apply_selection`` reuses the exogenous draw behind the OBSERVED
|
|
286
|
+
# ``prior_underwriter_score`` column, so low-severity selection is largely
|
|
287
|
+
# explainable from the feature matrix (corr ~0.92). The loop-on-SCM path
|
|
288
|
+
# sets this True to use a dedicated frozen draw instead, restoring the flat
|
|
289
|
+
# generator's semantics (severity 0 == selection-at-random that NO
|
|
290
|
+
# propensity model can explain). Drawn after every other node so the
|
|
291
|
+
# feature stream — and the fidelity-gated default path — is byte-identical.
|
|
292
|
+
if not 0.0 <= selection_severity <= 1.0:
|
|
293
|
+
raise ValueError(f"selection_severity must be in [0, 1]; got {selection_severity}")
|
|
294
|
+
if not 0.0 < approval_rate < 1.0:
|
|
295
|
+
raise ValueError(f"approval_rate must be in (0, 1); got {approval_rate}")
|
|
296
|
+
self.n_applicants = n_applicants
|
|
297
|
+
self.selection_severity = selection_severity
|
|
298
|
+
self.approval_rate = approval_rate
|
|
299
|
+
self.target_base_rate = target_base_rate
|
|
300
|
+
self.bank_feed_rate = bank_feed_rate
|
|
301
|
+
self.unobserved_strength = unobserved_strength
|
|
302
|
+
self.seed = seed
|
|
303
|
+
self.independent_selection_noise = independent_selection_noise
|
|
304
|
+
self.rng = np.random.Generator(np.random.PCG64(seed))
|
|
305
|
+
|
|
306
|
+
# ------------------------------------------------------------------ #
|
|
307
|
+
# Presets (mirror synthetic.py)
|
|
308
|
+
# ------------------------------------------------------------------ #
|
|
309
|
+
|
|
310
|
+
@classmethod
|
|
311
|
+
def unit(cls, seed: int = config.RANDOM_SEED, **kwargs) -> "StructuralBorrowerGenerator":
|
|
312
|
+
return cls(n_applicants=1500, seed=seed, **kwargs)
|
|
313
|
+
|
|
314
|
+
@classmethod
|
|
315
|
+
def benchmark(cls, seed: int = config.RANDOM_SEED, **kwargs) -> "StructuralBorrowerGenerator":
|
|
316
|
+
return cls(n_applicants=8000, seed=seed, **kwargs)
|
|
317
|
+
|
|
318
|
+
# ------------------------------------------------------------------ #
|
|
319
|
+
# DAG topology accessors (public; for downstream g-computation)
|
|
320
|
+
# ------------------------------------------------------------------ #
|
|
321
|
+
|
|
322
|
+
@classmethod
|
|
323
|
+
def dag_children(cls) -> dict[str, list[str]]:
|
|
324
|
+
"""Public DAG topology: ``node -> list of its direct child nodes``.
|
|
325
|
+
|
|
326
|
+
Derived from the modeled structural edges (``_CHILDREN``). A downstream
|
|
327
|
+
g-computation / standardization estimator can fit each child mechanism
|
|
328
|
+
``child ~ parents`` without touching private internals. Returns a fresh
|
|
329
|
+
copy with sorted, de-duplicated child lists.
|
|
330
|
+
"""
|
|
331
|
+
return {node: sorted(set(children))
|
|
332
|
+
for node, children in cls._CHILDREN.items()}
|
|
333
|
+
|
|
334
|
+
@classmethod
|
|
335
|
+
def dag_parents(cls) -> dict[str, list[str]]:
|
|
336
|
+
"""Public DAG topology: ``node -> list of its direct parent nodes``.
|
|
337
|
+
|
|
338
|
+
Exact inverse of :meth:`dag_children` over the same modeled feature nodes,
|
|
339
|
+
so every child lists the node as a parent (and vice-versa). Nodes with no
|
|
340
|
+
parents map to an empty list.
|
|
341
|
+
"""
|
|
342
|
+
parents: dict[str, list[str]] = {node: [] for node in cls._CHILDREN}
|
|
343
|
+
for node, children in cls._CHILDREN.items():
|
|
344
|
+
for child in children:
|
|
345
|
+
parents.setdefault(child, [])
|
|
346
|
+
if node not in parents[child]:
|
|
347
|
+
parents[child].append(node)
|
|
348
|
+
return {node: sorted(set(ps)) for node, ps in parents.items()}
|
|
349
|
+
|
|
350
|
+
# ------------------------------------------------------------------ #
|
|
351
|
+
# Public API
|
|
352
|
+
# ------------------------------------------------------------------ #
|
|
353
|
+
|
|
354
|
+
def generate_cohort(self) -> dict:
|
|
355
|
+
"""Return a cohort dict (SUPERSET of the synthetic contract).
|
|
356
|
+
|
|
357
|
+
Keys
|
|
358
|
+
----
|
|
359
|
+
features : pd.DataFrame[FEATURE_COLUMNS] (bank-feed cols NaN when no feed)
|
|
360
|
+
true_default : np.ndarray[int] planted default for ALL units
|
|
361
|
+
approved : np.ndarray[bool] prior-policy funding mask
|
|
362
|
+
prior_score : np.ndarray[float] selection score (lower = safer)
|
|
363
|
+
true_pd : np.ndarray[float] baseline SCM true default prob
|
|
364
|
+
days_to_default : np.ndarray[float] in [1,90] for defaulters, NaN else
|
|
365
|
+
observation_status : np.ndarray[object] 'matured' if approved else None
|
|
366
|
+
scm_state : SCMState frozen SCM state for counterfactuals
|
|
367
|
+
ground_truth : dict severity/base_rate/approval_rate/...
|
|
368
|
+
"""
|
|
369
|
+
state = self._build_state()
|
|
370
|
+
df = self._assemble_features(state)
|
|
371
|
+
|
|
372
|
+
true_pd = self._logit_to_pd(self._risk_logit(state.values, state))
|
|
373
|
+
# Planted realized default uses a frozen per-unit uniform draw.
|
|
374
|
+
default_u = state.noise["__default__"]
|
|
375
|
+
true_default = (default_u < true_pd).astype(int)
|
|
376
|
+
|
|
377
|
+
approved, prior_score = self._apply_selection(state, true_pd)
|
|
378
|
+
days_to_default = self._draw_survival(state, true_default)
|
|
379
|
+
observation_status = np.where(approved, "matured", None).astype(object)
|
|
380
|
+
|
|
381
|
+
return {
|
|
382
|
+
"features": df,
|
|
383
|
+
"true_default": true_default,
|
|
384
|
+
"approved": approved,
|
|
385
|
+
"prior_score": prior_score,
|
|
386
|
+
"true_pd": true_pd,
|
|
387
|
+
"days_to_default": days_to_default,
|
|
388
|
+
"observation_status": observation_status,
|
|
389
|
+
"scm_state": state,
|
|
390
|
+
"ground_truth": {
|
|
391
|
+
"selection_severity": self.selection_severity,
|
|
392
|
+
"base_rate": float(true_default.mean()),
|
|
393
|
+
"true_pd_mean": float(true_pd.mean()),
|
|
394
|
+
"approval_rate": float(approved.mean()),
|
|
395
|
+
"bank_feed_rate": float(state.has_feed.mean()),
|
|
396
|
+
"seed": self.seed,
|
|
397
|
+
"n_applicants": self.n_applicants,
|
|
398
|
+
},
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
def true_pd(self, state: SCMState | None = None) -> np.ndarray:
|
|
402
|
+
"""Baseline per-applicant TRUE default probability from the SCM.
|
|
403
|
+
|
|
404
|
+
With no ``state`` a fresh cohort is built. Shape ``(n,)`` float in [0,1].
|
|
405
|
+
"""
|
|
406
|
+
if state is None:
|
|
407
|
+
state = self._build_state()
|
|
408
|
+
return self._logit_to_pd(self._risk_logit(state.values, state))
|
|
409
|
+
|
|
410
|
+
def do_intervention(self, state: SCMState, feature: str, value) -> InterventionResult:
|
|
411
|
+
"""TRUE post-intervention default probability of ``do(feature = value)``.
|
|
412
|
+
|
|
413
|
+
Propagates ``value`` through the SCM: descendants of ``feature`` are
|
|
414
|
+
regenerated with the SAME frozen noise; ancestors and the unobserved
|
|
415
|
+
confounder are untouched. ``do(feature = observed_value)`` reproduces the
|
|
416
|
+
baseline exactly (per-unit no-op invariance). The bank-feed block is a
|
|
417
|
+
structural switch (toggling ``has_linked_bank_feed`` regenerates/deletes
|
|
418
|
+
the whole block). Returns an :class:`InterventionResult`.
|
|
419
|
+
|
|
420
|
+
``value`` is broadcast to all units (scalar) or used per-unit (length-n
|
|
421
|
+
array). Non-intervenable identity features (``sector``, ``vintage_years``,
|
|
422
|
+
...) are refused (``intervenable=False``, baseline returned unchanged).
|
|
423
|
+
``has_linked_bank_feed`` is a *structural* (information) switch: it is NOT
|
|
424
|
+
one of the 16 policy-intervenable features, but it IS manipulable — flipping
|
|
425
|
+
it reveals/hides the 6-node bank-feed block and recomputes the risk logit
|
|
426
|
+
through the existing structural equations (incl. the no-feed risk term).
|
|
427
|
+
Such results carry ``intervenable=False`` and ``structural=True``.
|
|
428
|
+
"""
|
|
429
|
+
baseline_pd = self._logit_to_pd(self._risk_logit(state.values, state))
|
|
430
|
+
|
|
431
|
+
in_support = self._check_support(feature, value)
|
|
432
|
+
|
|
433
|
+
# ---- structural (information) switch: has_linked_bank_feed -------------
|
|
434
|
+
if feature == "has_linked_bank_feed":
|
|
435
|
+
new_has_feed = self._coerce_feed(value, state.n)
|
|
436
|
+
post_pd = self._logit_to_pd(
|
|
437
|
+
self._risk_logit(state.values, state, has_feed=new_has_feed)
|
|
438
|
+
)
|
|
439
|
+
is_noop = bool(np.array_equal(new_has_feed, state.has_feed))
|
|
440
|
+
return InterventionResult(
|
|
441
|
+
feature=feature, value=_scalar(value), true_pd=post_pd,
|
|
442
|
+
baseline_pd=baseline_pd, effect=post_pd - baseline_pd,
|
|
443
|
+
intervenable=False, in_support=in_support, is_noop=is_noop,
|
|
444
|
+
structural=True,
|
|
445
|
+
note="structural bank-feed switch: information node toggled "
|
|
446
|
+
"(no-feed risk term re-applied; block revealed/hidden)",
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
intervenable = feature in INTERVENABLE_FEATURES
|
|
450
|
+
if not intervenable:
|
|
451
|
+
return InterventionResult(
|
|
452
|
+
feature=feature, value=_scalar(value), true_pd=baseline_pd.copy(),
|
|
453
|
+
baseline_pd=baseline_pd, effect=np.zeros_like(baseline_pd),
|
|
454
|
+
intervenable=False, in_support=in_support, is_noop=False,
|
|
455
|
+
note="non-intervenable feature: intervention refused (baseline returned)",
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
new_values = self._propagate(state, feature, value)
|
|
459
|
+
post_pd = self._logit_to_pd(self._risk_logit(new_values, state))
|
|
460
|
+
is_noop = bool(np.allclose(post_pd, baseline_pd, atol=1e-12))
|
|
461
|
+
return InterventionResult(
|
|
462
|
+
feature=feature, value=_scalar(value), true_pd=post_pd,
|
|
463
|
+
baseline_pd=baseline_pd, effect=post_pd - baseline_pd,
|
|
464
|
+
intervenable=True, in_support=in_support, is_noop=is_noop,
|
|
465
|
+
note="" if in_support else "value outside train [min,max] (extrapolation)",
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
@staticmethod
|
|
469
|
+
def _coerce_feed(value, n: int) -> np.ndarray:
|
|
470
|
+
"""Coerce a feed-switch ``value`` (scalar bool/0-1 or length-n array) to a
|
|
471
|
+
boolean ndarray of length ``n`` (truthy => feed linked)."""
|
|
472
|
+
v = np.asarray(value)
|
|
473
|
+
flags = np.broadcast_to(v, (n,)).astype(float)
|
|
474
|
+
return flags > 0.5
|
|
475
|
+
|
|
476
|
+
# ------------------------------------------------------------------ #
|
|
477
|
+
# SCM construction: layered latents -> fitted marginals
|
|
478
|
+
# ------------------------------------------------------------------ #
|
|
479
|
+
|
|
480
|
+
def _build_state(self) -> SCMState:
|
|
481
|
+
"""Sample the full SCM once; freeze noise + realized values on an SCMState."""
|
|
482
|
+
rng = self.rng
|
|
483
|
+
n = self.n_applicants
|
|
484
|
+
st = SCMState(n=n, seed=self.seed, generator=self)
|
|
485
|
+
st.has_feed = rng.random(n) < self.bank_feed_rate
|
|
486
|
+
|
|
487
|
+
# ---- exogenous standard-normal noise per node (frozen) ----
|
|
488
|
+
nodes = [
|
|
489
|
+
"sector", "geography_region", "vintage_years", "employee_count_bucket",
|
|
490
|
+
"stated_annual_revenue", "stated_time_in_business", "requested_amount",
|
|
491
|
+
"observed_monthly_revenue_avg_3mo", "observed_revenue_trend_3mo",
|
|
492
|
+
"observed_revenue_volatility", "observed_cash_balance_p10",
|
|
493
|
+
"observed_overdraft_count_3mo", "payroll_regularity_score",
|
|
494
|
+
"aggregate_credit_utilization", "recent_inquiries_count_6mo",
|
|
495
|
+
"existing_debt_obligations", "owner_personal_credit_band",
|
|
496
|
+
"invoice_payment_delinquency_rate", "prior_loans_count",
|
|
497
|
+
"application_channel", "multi_lender_inquiry_count_30d",
|
|
498
|
+
"prior_underwriter_score",
|
|
499
|
+
]
|
|
500
|
+
for node in nodes:
|
|
501
|
+
st.noise[node] = rng.standard_normal(n)
|
|
502
|
+
# Extra frozen draws (realized-default uniform, survival, prior-loan defaults).
|
|
503
|
+
st.noise["__default__"] = rng.random(n)
|
|
504
|
+
st.noise["__survival_body__"] = rng.random(n)
|
|
505
|
+
st.noise["__survival_mass__"] = rng.random(n)
|
|
506
|
+
st.noise["__prior_default__"] = rng.random(n)
|
|
507
|
+
st.confounder_u = rng.standard_normal(n)
|
|
508
|
+
if self.independent_selection_noise:
|
|
509
|
+
# Dedicated unobservable draw for the selection blend, drawn LAST and
|
|
510
|
+
# only when requested so the default path's RNG stream is untouched.
|
|
511
|
+
st.noise["__selection__"] = rng.standard_normal(n)
|
|
512
|
+
|
|
513
|
+
self._populate_values(st)
|
|
514
|
+
self._fit_risk_scaler(st)
|
|
515
|
+
return st
|
|
516
|
+
|
|
517
|
+
# latent helper: latent z = sum(beta_i * parent_latent_i) + sqrt(resid)*noise
|
|
518
|
+
@staticmethod
|
|
519
|
+
def _latent(noise, parents=(), resid_sd=None):
|
|
520
|
+
z = np.zeros_like(noise)
|
|
521
|
+
explained = 0.0
|
|
522
|
+
for beta, pz in parents:
|
|
523
|
+
z = z + beta * pz
|
|
524
|
+
explained += beta * beta # parents are ~unit-variance latents
|
|
525
|
+
if resid_sd is None:
|
|
526
|
+
resid_sd = np.sqrt(max(1.0 - explained, 1e-6))
|
|
527
|
+
return z + resid_sd * noise
|
|
528
|
+
|
|
529
|
+
def _populate_values(self, st: SCMState) -> None:
|
|
530
|
+
"""Fill ``st.latent`` and ``st.values`` for every node (ungated block)."""
|
|
531
|
+
L, V, N = st.latent, st.values, st.noise
|
|
532
|
+
|
|
533
|
+
# ---------------- business_identity roots (exogenous) ----------------
|
|
534
|
+
L["sector"] = N["sector"]
|
|
535
|
+
V["sector"] = _q_categorical(stats.norm.cdf(L["sector"]),
|
|
536
|
+
[.30095, .24858, .20151, .14938, .09958])
|
|
537
|
+
L["geography_region"] = N["geography_region"]
|
|
538
|
+
V["geography_region"] = _q_categorical(stats.norm.cdf(L["geography_region"]),
|
|
539
|
+
[.24701, .25337, .25096, .24865])
|
|
540
|
+
L["employee_count_bucket"] = N["employee_count_bucket"]
|
|
541
|
+
V["employee_count_bucket"] = _q_categorical(stats.norm.cdf(L["employee_count_bucket"]),
|
|
542
|
+
[.40014, .29820, .20177, .09989])
|
|
543
|
+
L["vintage_years"] = N["vintage_years"]
|
|
544
|
+
V["vintage_years"] = _q_gamma(stats.norm.cdf(L["vintage_years"]), 2.0033, 2.9799)
|
|
545
|
+
|
|
546
|
+
emp = L["employee_count_bucket"]
|
|
547
|
+
vin = L["vintage_years"]
|
|
548
|
+
sec = L["sector"]
|
|
549
|
+
|
|
550
|
+
# ---------------- self_reported ----------------
|
|
551
|
+
# stated_annual_revenue <- employee_count_bucket (eta .332)
|
|
552
|
+
L["stated_annual_revenue"] = self._latent(N["stated_annual_revenue"],
|
|
553
|
+
parents=[(0.55, emp)])
|
|
554
|
+
V["stated_annual_revenue"] = _q_lognormal(stats.norm.cdf(L["stated_annual_revenue"]),
|
|
555
|
+
14.444, 0.6137)
|
|
556
|
+
# stated_time_in_business == vintage echoed (r .996)
|
|
557
|
+
L["stated_time_in_business"] = self._latent(N["stated_time_in_business"],
|
|
558
|
+
parents=[(0.996, vin)], resid_sd=0.0894)
|
|
559
|
+
V["stated_time_in_business"] = _q_gamma(stats.norm.cdf(L["stated_time_in_business"]),
|
|
560
|
+
2.4615, 2.6006)
|
|
561
|
+
# requested_amount <- sector (eta .183), stated_revenue (confounder r +.635)
|
|
562
|
+
L["requested_amount"] = self._latent(N["requested_amount"],
|
|
563
|
+
parents=[(0.30, sec), (0.55, L["stated_annual_revenue"])])
|
|
564
|
+
V["requested_amount"] = _q_normal(stats.norm.cdf(L["requested_amount"]),
|
|
565
|
+
24923.5, 6153.0, lo=7590.0)
|
|
566
|
+
|
|
567
|
+
# ---------------- bank_feed GATED block ----------------
|
|
568
|
+
# observed_monthly_revenue_avg_3mo <- employee_count_bucket (eta .368),
|
|
569
|
+
# stated_revenue (r +.917 near-identity)
|
|
570
|
+
L["observed_monthly_revenue_avg_3mo"] = self._latent(
|
|
571
|
+
N["observed_monthly_revenue_avg_3mo"],
|
|
572
|
+
parents=[(0.30, emp), (0.62, L["stated_annual_revenue"])])
|
|
573
|
+
V["observed_monthly_revenue_avg_3mo"] = _q_lognormal(
|
|
574
|
+
stats.norm.cdf(L["observed_monthly_revenue_avg_3mo"]), 11.9729, 0.6063)
|
|
575
|
+
# intra-block covariance: payroll<->cash +.552, cash<->overdraft -.318,
|
|
576
|
+
# trend<->volatility -.361, payroll<->revenue +.454
|
|
577
|
+
L["payroll_regularity_score"] = self._latent(
|
|
578
|
+
N["payroll_regularity_score"],
|
|
579
|
+
parents=[(0.45, L["observed_monthly_revenue_avg_3mo"])])
|
|
580
|
+
V["payroll_regularity_score"] = _q_beta(
|
|
581
|
+
stats.norm.cdf(L["payroll_regularity_score"]), 2.6766, 2.4943)
|
|
582
|
+
L["observed_cash_balance_p10"] = self._latent(
|
|
583
|
+
N["observed_cash_balance_p10"],
|
|
584
|
+
parents=[(0.55, L["payroll_regularity_score"])])
|
|
585
|
+
V["observed_cash_balance_p10"] = _q_normal(
|
|
586
|
+
stats.norm.cdf(L["observed_cash_balance_p10"]), 459.92, 3038.44)
|
|
587
|
+
L["observed_overdraft_count_3mo"] = self._latent(
|
|
588
|
+
N["observed_overdraft_count_3mo"],
|
|
589
|
+
parents=[(-0.32, L["observed_cash_balance_p10"])])
|
|
590
|
+
V["observed_overdraft_count_3mo"] = _q_poisson(
|
|
591
|
+
stats.norm.cdf(L["observed_overdraft_count_3mo"]), 0.458)
|
|
592
|
+
L["observed_revenue_trend_3mo"] = self._latent(N["observed_revenue_trend_3mo"])
|
|
593
|
+
V["observed_revenue_trend_3mo"] = _q_normal(
|
|
594
|
+
stats.norm.cdf(L["observed_revenue_trend_3mo"]), 0.1508, 0.8218)
|
|
595
|
+
L["observed_revenue_volatility"] = self._latent(
|
|
596
|
+
N["observed_revenue_volatility"],
|
|
597
|
+
parents=[(-0.36, L["observed_revenue_trend_3mo"])])
|
|
598
|
+
V["observed_revenue_volatility"] = _q_lognormal(
|
|
599
|
+
stats.norm.cdf(L["observed_revenue_volatility"]), -0.0178, 0.5407)
|
|
600
|
+
|
|
601
|
+
# ---------------- bureau_credit ----------------
|
|
602
|
+
# aggregate_credit_utilization <- employee (eta .210), sector (eta .111),
|
|
603
|
+
# vintage (r -.291)
|
|
604
|
+
L["aggregate_credit_utilization"] = self._latent(
|
|
605
|
+
N["aggregate_credit_utilization"],
|
|
606
|
+
parents=[(0.21, emp), (0.11, sec), (-0.29, vin)])
|
|
607
|
+
V["aggregate_credit_utilization"] = _q_beta(
|
|
608
|
+
stats.norm.cdf(L["aggregate_credit_utilization"]), 1.7944, 1.7908)
|
|
609
|
+
L["recent_inquiries_count_6mo"] = self._latent(N["recent_inquiries_count_6mo"])
|
|
610
|
+
V["recent_inquiries_count_6mo"] = _q_poisson(
|
|
611
|
+
stats.norm.cdf(L["recent_inquiries_count_6mo"]), 0.6001)
|
|
612
|
+
# existing_debt_obligations <- employee (eta .183)
|
|
613
|
+
L["existing_debt_obligations"] = self._latent(
|
|
614
|
+
N["existing_debt_obligations"], parents=[(0.30, emp)])
|
|
615
|
+
V["existing_debt_obligations"] = _q_lognormal(
|
|
616
|
+
stats.norm.cdf(L["existing_debt_obligations"]), 9.026, 0.51)
|
|
617
|
+
# owner_personal_credit_band <- employee (eta .188); strongest categorical
|
|
618
|
+
L["owner_personal_credit_band"] = self._latent(
|
|
619
|
+
N["owner_personal_credit_band"], parents=[(0.30, emp)])
|
|
620
|
+
V["owner_personal_credit_band"] = _q_categorical(
|
|
621
|
+
stats.norm.cdf(L["owner_personal_credit_band"]),
|
|
622
|
+
[.20207, .20006, .20060, .19943, .19784])
|
|
623
|
+
|
|
624
|
+
# ---------------- platform_engagement ----------------
|
|
625
|
+
# invoice_payment_delinquency_rate <- vintage (r -.281), utilization (mediator r +.706)
|
|
626
|
+
L["invoice_payment_delinquency_rate"] = self._latent(
|
|
627
|
+
N["invoice_payment_delinquency_rate"],
|
|
628
|
+
parents=[(-0.28, vin), (0.55, L["aggregate_credit_utilization"])])
|
|
629
|
+
V["invoice_payment_delinquency_rate"] = _q_beta(
|
|
630
|
+
stats.norm.cdf(L["invoice_payment_delinquency_rate"]), 1.6859, 6.0527)
|
|
631
|
+
L["prior_loans_count"] = self._latent(N["prior_loans_count"])
|
|
632
|
+
V["prior_loans_count"] = _q_poisson(stats.norm.cdf(L["prior_loans_count"]), 0.3956)
|
|
633
|
+
# prior_loans_default_count ~ Binomial(count, 0.135); frozen via uniform.
|
|
634
|
+
cnt = V["prior_loans_count"].astype(int)
|
|
635
|
+
V["prior_loans_default_count"] = stats.binom.ppf(
|
|
636
|
+
N["__prior_default__"], np.maximum(cnt, 0), 0.135).astype(float)
|
|
637
|
+
|
|
638
|
+
# ---------------- application_context ----------------
|
|
639
|
+
L["application_channel"] = N["application_channel"]
|
|
640
|
+
V["application_channel"] = _q_categorical(
|
|
641
|
+
stats.norm.cdf(L["application_channel"]), [.49764, .35128, .15108])
|
|
642
|
+
L["multi_lender_inquiry_count_30d"] = self._latent(N["multi_lender_inquiry_count_30d"])
|
|
643
|
+
V["multi_lender_inquiry_count_30d"] = _q_poisson(
|
|
644
|
+
stats.norm.cdf(L["multi_lender_inquiry_count_30d"]), 0.4500)
|
|
645
|
+
|
|
646
|
+
# ---------------- derived ratios (deterministic) ----------------
|
|
647
|
+
self._recompute_derived(V)
|
|
648
|
+
|
|
649
|
+
# ---------------- prior_underwriter (selection) ----------------
|
|
650
|
+
# prior_underwriter_score: U-shaped beta, tilted by risk drivers (propensity).
|
|
651
|
+
L["prior_underwriter_score"] = self._latent(
|
|
652
|
+
N["prior_underwriter_score"],
|
|
653
|
+
parents=[(-0.10, L["aggregate_credit_utilization"]),
|
|
654
|
+
(-0.10, L["invoice_payment_delinquency_rate"]),
|
|
655
|
+
(0.10, L["requested_amount"])])
|
|
656
|
+
V["prior_underwriter_score"] = _q_beta(
|
|
657
|
+
stats.norm.cdf(L["prior_underwriter_score"]), 0.3852, 0.3818)
|
|
658
|
+
# prior_decision: hard threshold ~0.273 reproduces 0.606 approve rate.
|
|
659
|
+
V["prior_decision"] = (V["prior_underwriter_score"] >= 0.273).astype(float)
|
|
660
|
+
appr = V["prior_decision"] > 0.5
|
|
661
|
+
prior_approved_amount = _q_gamma(
|
|
662
|
+
stats.norm.cdf(N["requested_amount"]), 15.3714, 1523.73, lo=7590.0)
|
|
663
|
+
prior_approved_amount = np.where(appr, prior_approved_amount, np.nan)
|
|
664
|
+
V["prior_approved_amount"] = prior_approved_amount
|
|
665
|
+
|
|
666
|
+
@staticmethod
|
|
667
|
+
def _recompute_derived(V: dict) -> None:
|
|
668
|
+
"""Deterministic derived nodes (never independent): ratio + debt-to-revenue."""
|
|
669
|
+
rev_monthly = np.where(np.isfinite(V["observed_monthly_revenue_avg_3mo"]),
|
|
670
|
+
V["observed_monthly_revenue_avg_3mo"],
|
|
671
|
+
V["stated_annual_revenue"] / 12.0)
|
|
672
|
+
V["requested_amount_to_observed_revenue"] = V["requested_amount"] / np.maximum(rev_monthly, 1.0)
|
|
673
|
+
V["debt_to_revenue"] = V["existing_debt_obligations"] / np.maximum(V["stated_annual_revenue"], 1.0)
|
|
674
|
+
|
|
675
|
+
# ------------------------------------------------------------------ #
|
|
676
|
+
# Default risk logit (the structural outcome equation)
|
|
677
|
+
# ------------------------------------------------------------------ #
|
|
678
|
+
|
|
679
|
+
#: Structural coefficients for the latent default logit. Signs + relative
|
|
680
|
+
#: magnitudes match the grounded corr-to-default ranking in the DAG spec.
|
|
681
|
+
_RISK_TERMS = {
|
|
682
|
+
"invoice_payment_delinquency_rate": +1.05,
|
|
683
|
+
"aggregate_credit_utilization": +0.95,
|
|
684
|
+
"observed_cash_balance_p10": -0.95,
|
|
685
|
+
"requested_amount_to_observed_revenue": +0.70,
|
|
686
|
+
"observed_revenue_volatility": +0.60,
|
|
687
|
+
"requested_amount": +0.55,
|
|
688
|
+
"payroll_regularity_score": -0.55,
|
|
689
|
+
"observed_overdraft_count_3mo": +0.50,
|
|
690
|
+
"existing_debt_obligations": +0.40,
|
|
691
|
+
"debt_to_revenue": +0.30,
|
|
692
|
+
"owner_personal_credit_band": -0.30,
|
|
693
|
+
"recent_inquiries_count_6mo": +0.25,
|
|
694
|
+
"vintage_years": -0.25,
|
|
695
|
+
"stated_annual_revenue": -0.13,
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
def _fit_risk_scaler(self, st: SCMState) -> None:
|
|
699
|
+
"""Freeze per-term (mean,std) on the baseline cohort so the risk logit is a
|
|
700
|
+
stable structural function, then solve the intercept for the base rate."""
|
|
701
|
+
for term in self._RISK_TERMS:
|
|
702
|
+
x = np.asarray(st.values[term], dtype=float)
|
|
703
|
+
x = np.where(np.isfinite(x), x, np.nan)
|
|
704
|
+
st.risk_scaler[term] = (float(np.nanmean(x)), float(np.nanstd(x)))
|
|
705
|
+
logit_no_int = self._risk_logit_raw(st.values, st)
|
|
706
|
+
# Confounder folded in (unobserved): drives true risk; not exposed to model.
|
|
707
|
+
logit_no_int = logit_no_int + self.unobserved_strength * st.confounder_u
|
|
708
|
+
st.risk_intercept = _solve_intercept(logit_no_int, self.target_base_rate)
|
|
709
|
+
|
|
710
|
+
def _risk_logit_raw(self, values: dict, st: SCMState, has_feed=None) -> np.ndarray:
|
|
711
|
+
logit = np.zeros(st.n)
|
|
712
|
+
for term, coef in self._RISK_TERMS.items():
|
|
713
|
+
mean, std = st.risk_scaler[term]
|
|
714
|
+
logit = logit + coef * _zfix(values[term], mean, std)
|
|
715
|
+
# No-feed rows are slightly riskier (structural-missingness signal). The
|
|
716
|
+
# feed flag is a structural switch: do(has_linked_bank_feed=...) overrides
|
|
717
|
+
# it so linking/unlinking a feed flips this no-feed term (modest direct,
|
|
718
|
+
# information-only effect) — everything else is unchanged.
|
|
719
|
+
feed = st.has_feed if has_feed is None else has_feed
|
|
720
|
+
logit = logit + 0.20 * (~feed).astype(float)
|
|
721
|
+
return logit
|
|
722
|
+
|
|
723
|
+
def _risk_logit(self, values: dict, st: SCMState, has_feed=None) -> np.ndarray:
|
|
724
|
+
return (self._risk_logit_raw(values, st, has_feed=has_feed)
|
|
725
|
+
+ self.unobserved_strength * st.confounder_u + st.risk_intercept)
|
|
726
|
+
|
|
727
|
+
def _logit_to_pd(self, logit: np.ndarray) -> np.ndarray:
|
|
728
|
+
return np.clip(_sigmoid(logit), 0.0, 1.0)
|
|
729
|
+
|
|
730
|
+
# ------------------------------------------------------------------ #
|
|
731
|
+
# Intervention propagation (descendants only; frozen noise)
|
|
732
|
+
# ------------------------------------------------------------------ #
|
|
733
|
+
|
|
734
|
+
#: Direct children per node (the DAG edges we propagate along).
|
|
735
|
+
_CHILDREN = {
|
|
736
|
+
"stated_annual_revenue": ["requested_amount", "observed_monthly_revenue_avg_3mo",
|
|
737
|
+
"debt_to_revenue", "requested_amount_to_observed_revenue"],
|
|
738
|
+
"requested_amount": ["requested_amount_to_observed_revenue"],
|
|
739
|
+
"observed_monthly_revenue_avg_3mo": ["payroll_regularity_score",
|
|
740
|
+
"requested_amount_to_observed_revenue"],
|
|
741
|
+
"payroll_regularity_score": ["observed_cash_balance_p10"],
|
|
742
|
+
"observed_cash_balance_p10": ["observed_overdraft_count_3mo"],
|
|
743
|
+
"observed_revenue_trend_3mo": ["observed_revenue_volatility"],
|
|
744
|
+
"aggregate_credit_utilization": ["invoice_payment_delinquency_rate"],
|
|
745
|
+
"existing_debt_obligations": ["debt_to_revenue"],
|
|
746
|
+
# leaf intervenables (no structural children beyond the risk logit):
|
|
747
|
+
"observed_overdraft_count_3mo": [],
|
|
748
|
+
"observed_revenue_volatility": [],
|
|
749
|
+
"recent_inquiries_count_6mo": [],
|
|
750
|
+
"owner_personal_credit_band": [],
|
|
751
|
+
"invoice_payment_delinquency_rate": [],
|
|
752
|
+
"application_channel": [],
|
|
753
|
+
"multi_lender_inquiry_count_30d": [],
|
|
754
|
+
"stated_time_in_business": [],
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
#: Latent-equation parents (beta, parent_node) used to recompute a child's
|
|
758
|
+
#: latent from updated parents — mirrors _populate_values exactly.
|
|
759
|
+
_LATENT_PARENTS = {
|
|
760
|
+
"requested_amount": [(0.30, "sector"), (0.55, "stated_annual_revenue")],
|
|
761
|
+
"observed_monthly_revenue_avg_3mo": [(0.30, "employee_count_bucket"),
|
|
762
|
+
(0.62, "stated_annual_revenue")],
|
|
763
|
+
"payroll_regularity_score": [(0.45, "observed_monthly_revenue_avg_3mo")],
|
|
764
|
+
"observed_cash_balance_p10": [(0.55, "payroll_regularity_score")],
|
|
765
|
+
"observed_overdraft_count_3mo": [(-0.32, "observed_cash_balance_p10")],
|
|
766
|
+
"observed_revenue_volatility": [(-0.36, "observed_revenue_trend_3mo")],
|
|
767
|
+
"invoice_payment_delinquency_rate": [(-0.28, "vintage_years"),
|
|
768
|
+
(0.55, "aggregate_credit_utilization")],
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
#: Marginal quantile transform per node (latent -> value).
|
|
772
|
+
_MARGINAL = {
|
|
773
|
+
"stated_annual_revenue": lambda z: _q_lognormal(stats.norm.cdf(z), 14.444, 0.6137),
|
|
774
|
+
"requested_amount": lambda z: _q_normal(stats.norm.cdf(z), 24923.5, 6153.0, lo=7590.0),
|
|
775
|
+
"observed_monthly_revenue_avg_3mo": lambda z: _q_lognormal(stats.norm.cdf(z), 11.9729, 0.6063),
|
|
776
|
+
"payroll_regularity_score": lambda z: _q_beta(stats.norm.cdf(z), 2.6766, 2.4943),
|
|
777
|
+
"observed_cash_balance_p10": lambda z: _q_normal(stats.norm.cdf(z), 459.92, 3038.44),
|
|
778
|
+
"observed_overdraft_count_3mo": lambda z: _q_poisson(stats.norm.cdf(z), 0.458),
|
|
779
|
+
"observed_revenue_trend_3mo": lambda z: _q_normal(stats.norm.cdf(z), 0.1508, 0.8218),
|
|
780
|
+
"observed_revenue_volatility": lambda z: _q_lognormal(stats.norm.cdf(z), -0.0178, 0.5407),
|
|
781
|
+
"aggregate_credit_utilization": lambda z: _q_beta(stats.norm.cdf(z), 1.7944, 1.7908),
|
|
782
|
+
"recent_inquiries_count_6mo": lambda z: _q_poisson(stats.norm.cdf(z), 0.6001),
|
|
783
|
+
"existing_debt_obligations": lambda z: _q_lognormal(stats.norm.cdf(z), 9.026, 0.51),
|
|
784
|
+
"owner_personal_credit_band": lambda z: _q_categorical(
|
|
785
|
+
stats.norm.cdf(z), [.20207, .20006, .20060, .19943, .19784]),
|
|
786
|
+
"invoice_payment_delinquency_rate": lambda z: _q_beta(stats.norm.cdf(z), 1.6859, 6.0527),
|
|
787
|
+
"application_channel": lambda z: _q_categorical(stats.norm.cdf(z), [.49764, .35128, .15108]),
|
|
788
|
+
"multi_lender_inquiry_count_30d": lambda z: _q_poisson(stats.norm.cdf(z), 0.4500),
|
|
789
|
+
"stated_time_in_business": lambda z: _q_gamma(stats.norm.cdf(z), 2.4615, 2.6006),
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
#: Resid SD for child latent recompute (matches _populate_values explained var).
|
|
793
|
+
_RESID_SD = {
|
|
794
|
+
"stated_time_in_business": 0.0894,
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
def _child_latent(self, node: str, st: SCMState, L: dict) -> np.ndarray:
|
|
798
|
+
"""Recompute node's latent from (possibly updated) parent latents + frozen noise."""
|
|
799
|
+
parents = [(b, L[p]) for b, p in self._LATENT_PARENTS.get(node, [])]
|
|
800
|
+
return self._latent(st.noise[node], parents=parents, resid_sd=self._RESID_SD.get(node))
|
|
801
|
+
|
|
802
|
+
def _propagate(self, st: SCMState, feature: str, value) -> dict:
|
|
803
|
+
"""Return a NEW values dict for do(feature = value); descendants updated."""
|
|
804
|
+
# The bank-feed structural switch is handled directly in do_intervention
|
|
805
|
+
# (it toggles has_feed and re-applies the no-feed risk term rather than
|
|
806
|
+
# altering any node value), so it never reaches descendant propagation.
|
|
807
|
+
if feature == "has_linked_bank_feed": # pragma: no cover - guarded upstream
|
|
808
|
+
return dict(st.values)
|
|
809
|
+
|
|
810
|
+
L = dict(st.latent)
|
|
811
|
+
V = dict(st.values)
|
|
812
|
+
|
|
813
|
+
# Clamp the target value; invert to a clamped latent so descendants react.
|
|
814
|
+
v = np.asarray(value, dtype=float)
|
|
815
|
+
V[feature] = np.broadcast_to(v, (st.n,)).astype(float).copy()
|
|
816
|
+
L[feature] = self._invert_to_latent(feature, V[feature])
|
|
817
|
+
|
|
818
|
+
# Breadth-first propagation through descendants in DAG order.
|
|
819
|
+
order = self._descendant_order(feature)
|
|
820
|
+
for node in order:
|
|
821
|
+
if node in ("requested_amount_to_observed_revenue", "debt_to_revenue"):
|
|
822
|
+
continue # derived, recomputed below
|
|
823
|
+
L[node] = self._child_latent(node, st, L)
|
|
824
|
+
if node in self._MARGINAL:
|
|
825
|
+
V[node] = self._MARGINAL[node](L[node])
|
|
826
|
+
|
|
827
|
+
# Derived ratios recomputed from current V (handles both derived children).
|
|
828
|
+
self._recompute_derived(V)
|
|
829
|
+
# Re-apply bank-feed gating for the risk logit: gated nulls don't change
|
|
830
|
+
# because the risk logit uses raw values; gating only affects the emitted
|
|
831
|
+
# DataFrame, not the latent risk (no-feed handled by has_feed flag).
|
|
832
|
+
return V
|
|
833
|
+
|
|
834
|
+
def _descendant_order(self, feature: str) -> list[str]:
|
|
835
|
+
"""Topologically ordered descendants of ``feature`` (excluding itself)."""
|
|
836
|
+
order: list[str] = []
|
|
837
|
+
seen = set()
|
|
838
|
+
# repeated relaxation along _CHILDREN until closure (DAG, small)
|
|
839
|
+
frontier = list(self._CHILDREN.get(feature, []))
|
|
840
|
+
# Process in a stable layered manner.
|
|
841
|
+
topo = ["stated_annual_revenue", "requested_amount",
|
|
842
|
+
"observed_monthly_revenue_avg_3mo", "payroll_regularity_score",
|
|
843
|
+
"observed_cash_balance_p10", "observed_overdraft_count_3mo",
|
|
844
|
+
"observed_revenue_trend_3mo", "observed_revenue_volatility",
|
|
845
|
+
"aggregate_credit_utilization", "invoice_payment_delinquency_rate",
|
|
846
|
+
"existing_debt_obligations",
|
|
847
|
+
"requested_amount_to_observed_revenue", "debt_to_revenue"]
|
|
848
|
+
# Collect full descendant set.
|
|
849
|
+
desc = set()
|
|
850
|
+
stack = list(self._CHILDREN.get(feature, []))
|
|
851
|
+
while stack:
|
|
852
|
+
c = stack.pop()
|
|
853
|
+
if c in desc:
|
|
854
|
+
continue
|
|
855
|
+
desc.add(c)
|
|
856
|
+
stack.extend(self._CHILDREN.get(c, []))
|
|
857
|
+
for node in topo:
|
|
858
|
+
if node in desc and node not in seen:
|
|
859
|
+
order.append(node)
|
|
860
|
+
seen.add(node)
|
|
861
|
+
return order
|
|
862
|
+
|
|
863
|
+
def _invert_to_latent(self, feature: str, value: np.ndarray) -> np.ndarray:
|
|
864
|
+
"""Invert a clamped value back to its latent Gaussian (for descendant flow).
|
|
865
|
+
|
|
866
|
+
Uses the fitted marginal CDF -> norm.ppf. For discrete/categorical nodes
|
|
867
|
+
the latent is approximate (mid-rank) but only feeds descendant *latents*;
|
|
868
|
+
the clamped value itself is exact in the risk logit.
|
|
869
|
+
"""
|
|
870
|
+
u = self._marginal_cdf(feature, value)
|
|
871
|
+
u = np.clip(u, 1e-9, 1 - 1e-9)
|
|
872
|
+
return stats.norm.ppf(u)
|
|
873
|
+
|
|
874
|
+
@staticmethod
|
|
875
|
+
def _marginal_cdf(feature: str, value: np.ndarray) -> np.ndarray:
|
|
876
|
+
v = np.asarray(value, dtype=float)
|
|
877
|
+
if feature == "stated_annual_revenue":
|
|
878
|
+
return stats.lognorm.cdf(v, 0.6137, scale=np.exp(14.444))
|
|
879
|
+
if feature == "requested_amount":
|
|
880
|
+
return stats.norm.cdf(v, 24923.5, 6153.0)
|
|
881
|
+
if feature == "observed_monthly_revenue_avg_3mo":
|
|
882
|
+
return stats.lognorm.cdf(v, 0.6063, scale=np.exp(11.9729))
|
|
883
|
+
if feature == "payroll_regularity_score":
|
|
884
|
+
return stats.beta.cdf(v, 2.6766, 2.4943)
|
|
885
|
+
if feature == "observed_cash_balance_p10":
|
|
886
|
+
return stats.norm.cdf(v, 459.92, 3038.44)
|
|
887
|
+
if feature == "observed_overdraft_count_3mo":
|
|
888
|
+
return stats.poisson.cdf(v, 0.458)
|
|
889
|
+
if feature == "observed_revenue_trend_3mo":
|
|
890
|
+
return stats.norm.cdf(v, 0.1508, 0.8218)
|
|
891
|
+
if feature == "observed_revenue_volatility":
|
|
892
|
+
return stats.lognorm.cdf(v, 0.5407, scale=np.exp(-0.0178))
|
|
893
|
+
if feature == "aggregate_credit_utilization":
|
|
894
|
+
return stats.beta.cdf(v, 1.7944, 1.7908)
|
|
895
|
+
if feature == "recent_inquiries_count_6mo":
|
|
896
|
+
return stats.poisson.cdf(v, 0.6001)
|
|
897
|
+
if feature == "existing_debt_obligations":
|
|
898
|
+
return stats.lognorm.cdf(v, 0.51, scale=np.exp(9.026))
|
|
899
|
+
if feature == "owner_personal_credit_band":
|
|
900
|
+
return np.clip((v + 0.5) / 5.0, 1e-6, 1 - 1e-6)
|
|
901
|
+
if feature == "invoice_payment_delinquency_rate":
|
|
902
|
+
return stats.beta.cdf(v, 1.6859, 6.0527)
|
|
903
|
+
if feature == "application_channel":
|
|
904
|
+
return np.clip((v + 0.5) / 3.0, 1e-6, 1 - 1e-6)
|
|
905
|
+
if feature == "multi_lender_inquiry_count_30d":
|
|
906
|
+
return stats.poisson.cdf(v, 0.4500)
|
|
907
|
+
if feature == "stated_time_in_business":
|
|
908
|
+
return stats.gamma.cdf(v, 2.4615, scale=2.6006)
|
|
909
|
+
return np.full_like(v, 0.5)
|
|
910
|
+
|
|
911
|
+
def _check_support(self, feature: str, value) -> bool:
|
|
912
|
+
if feature not in FEATURE_SUPPORT:
|
|
913
|
+
return True
|
|
914
|
+
lo, hi = FEATURE_SUPPORT[feature]
|
|
915
|
+
v = np.asarray(value, dtype=float)
|
|
916
|
+
return bool(np.all((v >= lo) & (v <= hi)))
|
|
917
|
+
|
|
918
|
+
# ------------------------------------------------------------------ #
|
|
919
|
+
# Feature DataFrame assembly (apply bank-feed gating)
|
|
920
|
+
# ------------------------------------------------------------------ #
|
|
921
|
+
|
|
922
|
+
def _assemble_features(self, st: SCMState) -> pd.DataFrame:
|
|
923
|
+
data = {}
|
|
924
|
+
for col in FEATURE_COLUMNS:
|
|
925
|
+
if col == "has_linked_bank_feed":
|
|
926
|
+
data[col] = st.has_feed.astype(float)
|
|
927
|
+
else:
|
|
928
|
+
data[col] = np.asarray(st.values[col], dtype=float)
|
|
929
|
+
df = pd.DataFrame(data, columns=FEATURE_COLUMNS)
|
|
930
|
+
# Structural missingness: bank-feed block null where no feed linked.
|
|
931
|
+
# DIAG(leak-fix a): the revenue-derived ratio is computed at draw time from
|
|
932
|
+
# the *ungated* bank-feed revenue, so no-feed rows would otherwise carry feed
|
|
933
|
+
# information the structural-missingness design says they cannot have. Gate it
|
|
934
|
+
# alongside the block. True risk is unaffected (it reads st.values, ungated).
|
|
935
|
+
gated = list(BANK_FEED_COLUMNS) + ["requested_amount_to_observed_revenue"]
|
|
936
|
+
df.loc[~st.has_feed, gated] = np.nan
|
|
937
|
+
return df
|
|
938
|
+
|
|
939
|
+
# ------------------------------------------------------------------ #
|
|
940
|
+
# Selection (prior underwriter) + survival
|
|
941
|
+
# ------------------------------------------------------------------ #
|
|
942
|
+
|
|
943
|
+
def _apply_selection(self, st: SCMState, true_pd: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
944
|
+
"""Fund the lowest-risk ``approval_rate`` fraction.
|
|
945
|
+
|
|
946
|
+
``prior_score`` blends standardized true risk (incl. the unobserved
|
|
947
|
+
confounder) with noise per ``selection_severity`` — so the loop's frontier
|
|
948
|
+
escalation still makes sense (sev 0 == random selection, sev 1 == approval
|
|
949
|
+
tracks full latent risk including the unobservable).
|
|
950
|
+
|
|
951
|
+
The default blend noise reuses the exogenous draw behind the observed
|
|
952
|
+
``prior_underwriter_score`` feature (kept for backward compatibility: the
|
|
953
|
+
fidelity gate is baselined on this stream). With
|
|
954
|
+
``independent_selection_noise=True`` a dedicated frozen draw is used so
|
|
955
|
+
sev-0 selection is genuinely unexplainable from observed columns.
|
|
956
|
+
"""
|
|
957
|
+
sev = self.selection_severity
|
|
958
|
+
logit = self._risk_logit(st.values, st) # includes confounder
|
|
959
|
+
z = _zfix(logit, float(np.mean(logit)), float(np.std(logit)))
|
|
960
|
+
if self.independent_selection_noise:
|
|
961
|
+
noise = st.noise["__selection__"] # frozen, never exposed as a feature
|
|
962
|
+
else:
|
|
963
|
+
noise = st.noise["prior_underwriter_score"] # also generates the observed column
|
|
964
|
+
prior_score = sev * z + (1.0 - sev) * noise
|
|
965
|
+
cutoff = np.quantile(prior_score, self.approval_rate)
|
|
966
|
+
approved = prior_score <= cutoff
|
|
967
|
+
return approved, prior_score
|
|
968
|
+
|
|
969
|
+
def _draw_survival(self, st: SCMState, true_default: np.ndarray) -> np.ndarray:
|
|
970
|
+
"""days_to_default in [1,90]: smooth body over [3,60] + point mass at 90.
|
|
971
|
+
|
|
972
|
+
Per the survival spec: ~22.5% of defaults land exactly at day 90 (trigger
|
|
973
|
+
3); (60,90) is empty; defaulters only. Non-defaulters -> NaN.
|
|
974
|
+
"""
|
|
975
|
+
n = st.n
|
|
976
|
+
days = np.full(n, np.nan)
|
|
977
|
+
is_def = true_default == 1
|
|
978
|
+
body_u = st.noise["__survival_body__"]
|
|
979
|
+
mass_u = st.noise["__survival_mass__"]
|
|
980
|
+
# body: gamma-ish shape over [3,60], median ~37
|
|
981
|
+
body = 3.0 + 57.0 * stats.beta.ppf(np.clip(body_u, 1e-6, 1 - 1e-6), 2.2, 2.3)
|
|
982
|
+
body = np.clip(np.round(body), 3, 60)
|
|
983
|
+
at_90 = mass_u < 0.225
|
|
984
|
+
dd = np.where(at_90, 90.0, body)
|
|
985
|
+
days[is_def] = dd[is_def]
|
|
986
|
+
return days
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def dag_children() -> dict[str, list[str]]:
|
|
990
|
+
"""Module-level accessor for the SCM DAG topology (``node -> children``).
|
|
991
|
+
|
|
992
|
+
Thin wrapper over :meth:`StructuralBorrowerGenerator.dag_children` so a
|
|
993
|
+
downstream estimator can import the topology without instantiating or touching
|
|
994
|
+
private internals: ``from cldd import dag_children``.
|
|
995
|
+
"""
|
|
996
|
+
return StructuralBorrowerGenerator.dag_children()
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def dag_parents() -> dict[str, list[str]]:
|
|
1000
|
+
"""Module-level accessor for the SCM DAG topology (``node -> parents``).
|
|
1001
|
+
|
|
1002
|
+
Thin wrapper over :meth:`StructuralBorrowerGenerator.dag_parents`. Consistent
|
|
1003
|
+
with :func:`dag_children`: every child lists the node as a parent.
|
|
1004
|
+
"""
|
|
1005
|
+
return StructuralBorrowerGenerator.dag_parents()
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _scalar(value):
|
|
1009
|
+
v = np.asarray(value, dtype=float)
|
|
1010
|
+
return float(v.reshape(-1)[0]) if v.size else float("nan")
|