creditdecide 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- creditdecide/__init__.py +40 -0
- creditdecide/_version.py +1 -0
- creditdecide/bootstrap.py +160 -0
- creditdecide/calibration.py +128 -0
- creditdecide/cli.py +377 -0
- creditdecide/data/__init__.py +0 -0
- creditdecide/data/credit_default.csv.gz +0 -0
- creditdecide/data/credit_default.json +17 -0
- creditdecide/datasets.py +198 -0
- creditdecide/decision.py +250 -0
- creditdecide/fairness.py +98 -0
- creditdecide/forest.py +27 -0
- creditdecide/metrics.py +35 -0
- creditdecide/modelcard.py +219 -0
- creditdecide/modelio.py +209 -0
- creditdecide/monitor.py +71 -0
- creditdecide/pipeline.py +552 -0
- creditdecide/plotting.py +64 -0
- creditdecide/preprocess.py +38 -0
- creditdecide/quality.py +61 -0
- creditdecide/reasons.py +188 -0
- creditdecide/sql/__init__.py +203 -0
- creditdecide/sql/class_counts.sql +5 -0
- creditdecide/sql/duplicates.sql +14 -0
- creditdecide/sql/gains.sql +37 -0
- creditdecide/sql/outcomes.sql +20 -0
- creditdecide/sql/outcomes_rows.sql +6 -0
- creditdecide/sql/psi.sql +44 -0
- creditdecide/sql/tableau_accounts.sql +25 -0
- creditdecide/sql/tableau_gains.sql +7 -0
- creditdecide/sql/tableau_models.sql +6 -0
- creditdecide/sql/tableau_reasons.sql +9 -0
- creditdecide/sql/undocumented_codes.sql +10 -0
- creditdecide/tableau.py +133 -0
- creditdecide/tuning.py +137 -0
- creditdecide-0.1.0.dist-info/METADATA +283 -0
- creditdecide-0.1.0.dist-info/RECORD +41 -0
- creditdecide-0.1.0.dist-info/WHEEL +4 -0
- creditdecide-0.1.0.dist-info/entry_points.txt +2 -0
- creditdecide-0.1.0.dist-info/licenses/DATA_LICENSE.md +26 -0
- creditdecide-0.1.0.dist-info/licenses/LICENSE +21 -0
creditdecide/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""creditdecide: from a labeled table to a monitored, explainable decision rule.
|
|
2
|
+
|
|
3
|
+
Starts with a data-quality report, then trains a logistic regression and a random forest
|
|
4
|
+
(optionally tuned, which adds gradient boosting). The best model is benchmarked against the best
|
|
5
|
+
one-feature rule, its probabilities are calibrated and turned into a budget- or cost-based
|
|
6
|
+
decision rule, every flagged row gets reason codes, the test results get bootstrap confidence
|
|
7
|
+
intervals, and the model is saved as a checksum-verified directory with drift monitoring and a
|
|
8
|
+
fairness audit. The data-quality checks, gains table, outcomes join and drift bins are DuckDB
|
|
9
|
+
SQL (creditdecide.sql), and ``creditdecide tableau`` writes a run's results as Tableau-ready
|
|
10
|
+
tables.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from creditdecide._version import __version__
|
|
14
|
+
from creditdecide.datasets import (
|
|
15
|
+
CREDIT_DEFAULT_SPEC,
|
|
16
|
+
FeatureSpec,
|
|
17
|
+
load_credit_default,
|
|
18
|
+
load_csv,
|
|
19
|
+
)
|
|
20
|
+
from creditdecide.decision import DecisionPolicy
|
|
21
|
+
from creditdecide.modelio import ScoringModel, load_model
|
|
22
|
+
from creditdecide.pipeline import MODELS, SCORING_MODELS, Config, Result, run
|
|
23
|
+
from creditdecide.quality import quality_report
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"CREDIT_DEFAULT_SPEC",
|
|
27
|
+
"MODELS",
|
|
28
|
+
"SCORING_MODELS",
|
|
29
|
+
"Config",
|
|
30
|
+
"DecisionPolicy",
|
|
31
|
+
"FeatureSpec",
|
|
32
|
+
"Result",
|
|
33
|
+
"ScoringModel",
|
|
34
|
+
"__version__",
|
|
35
|
+
"load_credit_default",
|
|
36
|
+
"load_csv",
|
|
37
|
+
"load_model",
|
|
38
|
+
"quality_report",
|
|
39
|
+
"run",
|
|
40
|
+
]
|
creditdecide/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Bootstrap confidence intervals for results measured on the test split.
|
|
2
|
+
|
|
3
|
+
The test split is one sample of customers (6,002 in the bundled data). A different month, or a
|
|
4
|
+
different random split, would give somewhat different numbers, so every headline figure gets
|
|
5
|
+
an interval.
|
|
6
|
+
|
|
7
|
+
How it works: test accounts are drawn with replacement, as many as there are, and every result
|
|
8
|
+
is recomputed on the new sample; this is repeated (1,000 times by default) and the 2.5th and
|
|
9
|
+
97.5th percentiles of the repeats form the 95% interval (the percentile bootstrap, Efron &
|
|
10
|
+
Tibshirani 1993, ch. 13). Rows with identical inputs are drawn together, matching the
|
|
11
|
+
group-aware split. Comparisons (model minus rule, one model minus another) are computed on the
|
|
12
|
+
same draw, so the interval of a difference accounts for the two results moving together.
|
|
13
|
+
|
|
14
|
+
What is held fixed: the trained models, the threshold, the calibration map and the chosen
|
|
15
|
+
rule. The intervals therefore measure how much the test figures depend on which customers
|
|
16
|
+
happened to be in the test sample. They do not include the variation from retraining on
|
|
17
|
+
different data or with another seed, which would widen them.
|
|
18
|
+
|
|
19
|
+
Instead of copying rows, each draw is a vector of weights: how many times each row was
|
|
20
|
+
drawn (0, 1, 2, ...). Weight 1 for every row reproduces the original figures exactly.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from collections.abc import Callable
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
import pandas as pd
|
|
29
|
+
|
|
30
|
+
DEFAULT_REPLICATES = 1000
|
|
31
|
+
LEVEL = 0.95
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def resample_weights(groups, n_boot: int, seed: int = 0):
|
|
35
|
+
"""Yield ``n_boot`` weight vectors, one per bootstrap sample.
|
|
36
|
+
|
|
37
|
+
``groups`` gives each row's group id; groups are drawn with replacement, as many as there
|
|
38
|
+
are, and every row of a group gets the group's draw count.
|
|
39
|
+
"""
|
|
40
|
+
g = pd.factorize(np.asarray(groups))[0]
|
|
41
|
+
n_groups = int(g.max()) + 1 if len(g) else 0
|
|
42
|
+
rng = np.random.default_rng(seed)
|
|
43
|
+
for _ in range(n_boot):
|
|
44
|
+
counts = np.bincount(rng.integers(0, n_groups, n_groups), minlength=n_groups)
|
|
45
|
+
yield counts[g].astype(float)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def capture_rate(w, y, flag) -> float:
|
|
49
|
+
"""Weighted share of positives that are flagged."""
|
|
50
|
+
pos = w * (np.asarray(y) == 1)
|
|
51
|
+
total = pos.sum()
|
|
52
|
+
return float((pos * flag).sum() / total) if total else float("nan")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def precision(w, y, flag) -> float:
|
|
56
|
+
"""Weighted share of flagged rows that are positive."""
|
|
57
|
+
f = w * np.asarray(flag, bool)
|
|
58
|
+
total = f.sum()
|
|
59
|
+
return float((f * (np.asarray(y) == 1)).sum() / total) if total else float("nan")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def flagged_fraction(w, flag) -> float:
|
|
63
|
+
return float((w * np.asarray(flag, bool)).sum() / w.sum())
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class RankedScores:
|
|
67
|
+
"""A score vector prepared once for fast weighted ROC AUC on many bootstrap samples."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, y, score):
|
|
70
|
+
self.pos = np.asarray(y) == 1
|
|
71
|
+
_, self.inverse = np.unique(np.asarray(score, float), return_inverse=True)
|
|
72
|
+
self.n_unique = int(self.inverse.max()) + 1 if len(self.inverse) else 0
|
|
73
|
+
|
|
74
|
+
def auc(self, w) -> float:
|
|
75
|
+
"""Weighted ROC AUC: the probability that a random positive outscores a random
|
|
76
|
+
negative, ties counting one half (the Mann-Whitney form)."""
|
|
77
|
+
pos = np.bincount(self.inverse, weights=w * self.pos, minlength=self.n_unique)
|
|
78
|
+
neg = np.bincount(self.inverse, weights=w * ~self.pos, minlength=self.n_unique)
|
|
79
|
+
below = np.cumsum(neg) - neg
|
|
80
|
+
denom = pos.sum() * neg.sum()
|
|
81
|
+
return float((pos * (below + 0.5 * neg)).sum() / denom) if denom else float("nan")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def weighted_roc_auc(y, score, w=None) -> float:
|
|
85
|
+
"""ROC AUC with row weights (equal to scikit-learn's roc_auc_score with sample_weight)."""
|
|
86
|
+
w = np.ones(len(y)) if w is None else np.asarray(w, float)
|
|
87
|
+
return RankedScores(y, score).auc(w)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def cost(w, y, flag, cost_fp: float, missed) -> float:
|
|
91
|
+
"""Weighted cost of a rule: cost_fp per flagged negative plus the missed cost of every
|
|
92
|
+
positive that is not flagged."""
|
|
93
|
+
y, flag = np.asarray(y) == 1, np.asarray(flag, bool)
|
|
94
|
+
return float(cost_fp * (w * (flag & ~y)).sum() + (w * missed * (~flag & y)).sum())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def saving_vs_best_trivial(w, y, flag, cost_fp: float, missed) -> float:
|
|
98
|
+
"""Cost saving against the better of flagging nobody and flagging everybody, as a
|
|
99
|
+
fraction of that strategy's cost (same definition as decision.summarize)."""
|
|
100
|
+
none = cost(w, y, np.zeros(len(y), bool), cost_fp, missed)
|
|
101
|
+
everyone = cost(w, y, np.ones(len(y), bool), cost_fp, missed)
|
|
102
|
+
best = min(none, everyone)
|
|
103
|
+
return float((best - cost(w, y, flag, cost_fp, missed)) / best) if best else 0.0
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def bootstrap(statistic: Callable[[np.ndarray], dict], groups, n_boot: int = DEFAULT_REPLICATES,
|
|
107
|
+
seed: int = 0, level: float = LEVEL, differences=(),
|
|
108
|
+
units: dict | None = None) -> pd.DataFrame:
|
|
109
|
+
"""Percentile bootstrap intervals for every number ``statistic`` returns.
|
|
110
|
+
|
|
111
|
+
``statistic(w)`` takes a vector of row weights and returns ``{name: value}``. Names listed
|
|
112
|
+
in ``differences`` are comparisons; for those the table says whether the interval excludes
|
|
113
|
+
zero, i.e. whether the sign of the difference is clear at this confidence level. ``units``
|
|
114
|
+
maps names to "fraction", "relative", "auc" or "ratio" (used only for printing; default
|
|
115
|
+
"fraction").
|
|
116
|
+
"""
|
|
117
|
+
units = units or {}
|
|
118
|
+
if n_boot < 1:
|
|
119
|
+
raise ValueError("n_boot must be at least 1")
|
|
120
|
+
if not 0 < level < 1:
|
|
121
|
+
raise ValueError("level must be between 0 and 1")
|
|
122
|
+
n = len(np.asarray(groups))
|
|
123
|
+
estimate = statistic(np.ones(n))
|
|
124
|
+
draws = {k: np.empty(n_boot) for k in estimate}
|
|
125
|
+
for b, w in enumerate(resample_weights(groups, n_boot, seed)):
|
|
126
|
+
for k, v in statistic(w).items():
|
|
127
|
+
draws[k][b] = v
|
|
128
|
+
tail = (1 - level) / 2
|
|
129
|
+
rows = []
|
|
130
|
+
for k, v in estimate.items():
|
|
131
|
+
d = draws[k]
|
|
132
|
+
lo, hi = np.nanquantile(d, [tail, 1 - tail])
|
|
133
|
+
is_diff = k in differences
|
|
134
|
+
rows.append({"quantity": k, "type": "difference" if is_diff else "estimate",
|
|
135
|
+
"unit": units.get(k, "fraction"),
|
|
136
|
+
"estimate": float(v), "ci_low": float(lo), "ci_high": float(hi),
|
|
137
|
+
"std_error": float(np.nanstd(d, ddof=1)) if n_boot > 1 else float("nan"),
|
|
138
|
+
"excludes_zero": bool(lo > 0 or hi < 0) if is_diff else None})
|
|
139
|
+
out = pd.DataFrame(rows)
|
|
140
|
+
out.attrs.update({"replicates": n_boot, "level": level, "seed": seed})
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def format_interval(row) -> str:
|
|
145
|
+
"""One line per quantity: estimate and interval in readable units. A difference of two
|
|
146
|
+
fractions is in percentage points; a relative error (unit "relative") in percent."""
|
|
147
|
+
unit, diff = row["unit"], row["type"] == "difference"
|
|
148
|
+
sign = "+" if diff else ""
|
|
149
|
+
if unit in ("fraction", "relative"):
|
|
150
|
+
suffix = " points" if diff and unit == "fraction" else "%"
|
|
151
|
+
est, lo, hi = (f"{100 * row[c]:{sign}.1f}" for c in ("estimate", "ci_low", "ci_high"))
|
|
152
|
+
text = (f"{est}{suffix} [{lo}, {hi}]" if suffix == " points"
|
|
153
|
+
else f"{est}% [{lo}%, {hi}%]")
|
|
154
|
+
else:
|
|
155
|
+
digits = 3 if unit == "auc" else 2
|
|
156
|
+
est, lo, hi = (f"{row[c]:{sign}.{digits}f}" for c in ("estimate", "ci_low", "ci_high"))
|
|
157
|
+
text = f"{est} [{lo}, {hi}]"
|
|
158
|
+
if diff:
|
|
159
|
+
text += "; sign clear" if row["excludes_zero"] else "; includes zero"
|
|
160
|
+
return f"{row['quantity']}: {text}"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Probability calibration.
|
|
2
|
+
|
|
3
|
+
A model can rank accounts well and still give probabilities that are too high or too low.
|
|
4
|
+
Ranking is enough to decide *which* accounts to act on; calibrated probabilities are needed to
|
|
5
|
+
answer *how many* defaults to expect (staffing, provisioning: expected loss = PD x LGD x
|
|
6
|
+
exposure) and to make per-account expected-cost decisions (creditdecide.decision).
|
|
7
|
+
|
|
8
|
+
Method
|
|
9
|
+
------
|
|
10
|
+
The scoring model is fixed. A one-dimensional map from its raw score to a probability is
|
|
11
|
+
fitted on the VALIDATION split, which the model never trained on:
|
|
12
|
+
|
|
13
|
+
* ``none`` identity;
|
|
14
|
+
* ``sigmoid`` logistic regression on the log-odds of the raw score (Platt scaling);
|
|
15
|
+
* ``isotonic`` monotone step function (pool-adjacent-violators).
|
|
16
|
+
|
|
17
|
+
The method is chosen by 5-fold stratified cross-validation *within* the validation split on log
|
|
18
|
+
loss (a proper scoring rule), with the one-standard-error rule: the simplest method (none, then
|
|
19
|
+
sigmoid, then isotonic) whose cross-validated log loss is within one standard error of the best
|
|
20
|
+
is used, so a map is only added when it clearly helps. It is then refitted on the whole
|
|
21
|
+
validation split. The test split is only used to report the result. Every map is monotone
|
|
22
|
+
non-decreasing, so the ranking of accounts, and therefore every threshold decision, is
|
|
23
|
+
unchanged by calibration.
|
|
24
|
+
|
|
25
|
+
The fitted map is stored as plain numbers (two coefficients, or the isotonic breakpoints) in the
|
|
26
|
+
model manifest and applied with ``np.interp``, so no extra code needs to be deserialized.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import numpy as np
|
|
32
|
+
import pandas as pd
|
|
33
|
+
from sklearn.isotonic import IsotonicRegression
|
|
34
|
+
from sklearn.linear_model import LogisticRegression
|
|
35
|
+
from sklearn.metrics import brier_score_loss, log_loss
|
|
36
|
+
from sklearn.model_selection import StratifiedKFold
|
|
37
|
+
|
|
38
|
+
METHODS = ("none", "sigmoid", "isotonic")
|
|
39
|
+
EPS = 1e-6
|
|
40
|
+
N_BINS = 10
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _logit(p):
|
|
44
|
+
p = np.clip(np.asarray(p, float), EPS, 1 - EPS)
|
|
45
|
+
return np.log(p / (1 - p))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def fit_map(method: str, score, y) -> dict:
|
|
49
|
+
"""Fit one calibration map; returns its JSON-serialisable parameters."""
|
|
50
|
+
score, y = np.asarray(score, float), np.asarray(y, int)
|
|
51
|
+
if method == "none":
|
|
52
|
+
return {"method": "none"}
|
|
53
|
+
if method == "sigmoid":
|
|
54
|
+
lr = LogisticRegression(C=1e6, max_iter=1000).fit(_logit(score)[:, None], y)
|
|
55
|
+
return {"method": "sigmoid", "a": float(lr.coef_[0, 0]), "b": float(lr.intercept_[0])}
|
|
56
|
+
if method == "isotonic":
|
|
57
|
+
iso = IsotonicRegression(out_of_bounds="clip", y_min=0.0, y_max=1.0).fit(score, y)
|
|
58
|
+
return {"method": "isotonic", "x": iso.X_thresholds_.tolist(),
|
|
59
|
+
"y": iso.y_thresholds_.tolist()}
|
|
60
|
+
raise ValueError(f"unknown calibration method {method!r}; choose from {METHODS}")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def apply_map(params: dict, score) -> np.ndarray:
|
|
64
|
+
score = np.asarray(score, float)
|
|
65
|
+
m = params["method"]
|
|
66
|
+
if m == "none":
|
|
67
|
+
return np.clip(score, 0.0, 1.0)
|
|
68
|
+
if m == "sigmoid":
|
|
69
|
+
return 1.0 / (1.0 + np.exp(-(params["a"] * _logit(score) + params["b"])))
|
|
70
|
+
if m == "isotonic":
|
|
71
|
+
return np.interp(score, params["x"], params["y"]) # constant beyond the end points
|
|
72
|
+
raise ValueError(f"unknown calibration method {m!r}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def choose_and_fit(score_val, y_val, method: str = "auto", seed: int = 0,
|
|
76
|
+
n_folds: int = 5) -> tuple[dict, pd.DataFrame]:
|
|
77
|
+
"""Pick a method by cross-validated log loss on the validation split (or use ``method``),
|
|
78
|
+
then fit it on the whole validation split. Returns (params, cross-validation table)."""
|
|
79
|
+
s, y = np.asarray(score_val, float), np.asarray(y_val, int)
|
|
80
|
+
rows = []
|
|
81
|
+
folds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
|
|
82
|
+
for m in METHODS:
|
|
83
|
+
ll, br = [], []
|
|
84
|
+
for fit_idx, ev_idx in folds.split(s, y):
|
|
85
|
+
p = apply_map(fit_map(m, s[fit_idx], y[fit_idx]), s[ev_idx])
|
|
86
|
+
ll.append(log_loss(y[ev_idx], np.clip(p, EPS, 1 - EPS), labels=[0, 1]))
|
|
87
|
+
br.append(brier_score_loss(y[ev_idx], p))
|
|
88
|
+
rows.append({"method": m, "cv_log_loss": float(np.mean(ll)),
|
|
89
|
+
"cv_log_loss_se": float(np.std(ll, ddof=1) / np.sqrt(n_folds)),
|
|
90
|
+
"cv_brier": float(np.mean(br))})
|
|
91
|
+
table = pd.DataFrame(rows)
|
|
92
|
+
if method == "auto": # one-standard-error rule; METHODS is ordered simplest first
|
|
93
|
+
best = table.loc[table.cv_log_loss.idxmin()]
|
|
94
|
+
ok = table.cv_log_loss <= best.cv_log_loss + best.cv_log_loss_se
|
|
95
|
+
method = str(table.loc[ok, "method"].iloc[0])
|
|
96
|
+
elif method not in METHODS:
|
|
97
|
+
raise ValueError(f"calibration must be 'auto' or one of {METHODS}")
|
|
98
|
+
table["chosen"] = table["method"] == method
|
|
99
|
+
return fit_map(method, s, y), table
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def reliability_table(y, p, n_bins: int = N_BINS) -> pd.DataFrame:
|
|
103
|
+
"""Equal-count bins of predicted probability: mean prediction vs observed rate."""
|
|
104
|
+
y, p = np.asarray(y, int), np.asarray(p, float)
|
|
105
|
+
order = np.argsort(p, kind="stable")
|
|
106
|
+
rows = []
|
|
107
|
+
for i, idx in enumerate(np.array_split(order, n_bins), start=1):
|
|
108
|
+
if len(idx) == 0:
|
|
109
|
+
continue
|
|
110
|
+
rows.append({"bin": i, "n": len(idx), "mean_predicted": float(p[idx].mean()),
|
|
111
|
+
"observed_rate": float(y[idx].mean())})
|
|
112
|
+
return pd.DataFrame(rows)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def calibration_metrics(y, p, n_bins: int = N_BINS) -> dict:
|
|
116
|
+
"""Brier score, log loss, expected calibration error (equal-count bins), and the
|
|
117
|
+
portfolio check: expected number of positives (sum of probabilities) vs actual."""
|
|
118
|
+
y, p = np.asarray(y, int), np.asarray(p, float)
|
|
119
|
+
t = reliability_table(y, p, n_bins)
|
|
120
|
+
ece = float(np.sum(t["n"] * (t["mean_predicted"] - t["observed_rate"]).abs()) / len(y))
|
|
121
|
+
return {
|
|
122
|
+
"brier": float(brier_score_loss(y, p)),
|
|
123
|
+
"log_loss": float(log_loss(y, np.clip(p, EPS, 1 - EPS), labels=[0, 1])),
|
|
124
|
+
"ece": ece,
|
|
125
|
+
"max_bin_gap": float((t["mean_predicted"] - t["observed_rate"]).abs().max()),
|
|
126
|
+
"expected_positives": float(p.sum()),
|
|
127
|
+
"actual_positives": int(y.sum()),
|
|
128
|
+
}
|