creditdecide 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. creditdecide/__init__.py +40 -0
  2. creditdecide/_version.py +1 -0
  3. creditdecide/bootstrap.py +160 -0
  4. creditdecide/calibration.py +128 -0
  5. creditdecide/cli.py +377 -0
  6. creditdecide/data/__init__.py +0 -0
  7. creditdecide/data/credit_default.csv.gz +0 -0
  8. creditdecide/data/credit_default.json +17 -0
  9. creditdecide/datasets.py +198 -0
  10. creditdecide/decision.py +250 -0
  11. creditdecide/fairness.py +98 -0
  12. creditdecide/forest.py +27 -0
  13. creditdecide/metrics.py +35 -0
  14. creditdecide/modelcard.py +219 -0
  15. creditdecide/modelio.py +209 -0
  16. creditdecide/monitor.py +71 -0
  17. creditdecide/pipeline.py +552 -0
  18. creditdecide/plotting.py +64 -0
  19. creditdecide/preprocess.py +38 -0
  20. creditdecide/quality.py +61 -0
  21. creditdecide/reasons.py +188 -0
  22. creditdecide/sql/__init__.py +203 -0
  23. creditdecide/sql/class_counts.sql +5 -0
  24. creditdecide/sql/duplicates.sql +14 -0
  25. creditdecide/sql/gains.sql +37 -0
  26. creditdecide/sql/outcomes.sql +20 -0
  27. creditdecide/sql/outcomes_rows.sql +6 -0
  28. creditdecide/sql/psi.sql +44 -0
  29. creditdecide/sql/tableau_accounts.sql +25 -0
  30. creditdecide/sql/tableau_gains.sql +7 -0
  31. creditdecide/sql/tableau_models.sql +6 -0
  32. creditdecide/sql/tableau_reasons.sql +9 -0
  33. creditdecide/sql/undocumented_codes.sql +10 -0
  34. creditdecide/tableau.py +133 -0
  35. creditdecide/tuning.py +137 -0
  36. creditdecide-0.1.0.dist-info/METADATA +283 -0
  37. creditdecide-0.1.0.dist-info/RECORD +41 -0
  38. creditdecide-0.1.0.dist-info/WHEEL +4 -0
  39. creditdecide-0.1.0.dist-info/entry_points.txt +2 -0
  40. creditdecide-0.1.0.dist-info/licenses/DATA_LICENSE.md +26 -0
  41. creditdecide-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,40 @@
1
+ """creditdecide: from a labeled table to a monitored, explainable decision rule.
2
+
3
+ Starts with a data-quality report, then trains a logistic regression and a random forest
4
+ (optionally tuned, which adds gradient boosting). The best model is benchmarked against the best
5
+ one-feature rule, its probabilities are calibrated and turned into a budget- or cost-based
6
+ decision rule, every flagged row gets reason codes, the test results get bootstrap confidence
7
+ intervals, and the model is saved as a checksum-verified directory with drift monitoring and a
8
+ fairness audit. The data-quality checks, gains table, outcomes join and drift bins are DuckDB
9
+ SQL (creditdecide.sql), and ``creditdecide tableau`` writes a run's results as Tableau-ready
10
+ tables.
11
+ """
12
+
13
+ from creditdecide._version import __version__
14
+ from creditdecide.datasets import (
15
+ CREDIT_DEFAULT_SPEC,
16
+ FeatureSpec,
17
+ load_credit_default,
18
+ load_csv,
19
+ )
20
+ from creditdecide.decision import DecisionPolicy
21
+ from creditdecide.modelio import ScoringModel, load_model
22
+ from creditdecide.pipeline import MODELS, SCORING_MODELS, Config, Result, run
23
+ from creditdecide.quality import quality_report
24
+
25
+ __all__ = [
26
+ "CREDIT_DEFAULT_SPEC",
27
+ "MODELS",
28
+ "SCORING_MODELS",
29
+ "Config",
30
+ "DecisionPolicy",
31
+ "FeatureSpec",
32
+ "Result",
33
+ "ScoringModel",
34
+ "__version__",
35
+ "load_credit_default",
36
+ "load_csv",
37
+ "load_model",
38
+ "quality_report",
39
+ "run",
40
+ ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,160 @@
1
+ """Bootstrap confidence intervals for results measured on the test split.
2
+
3
+ The test split is one sample of customers (6,002 in the bundled data). A different month, or a
4
+ different random split, would give somewhat different numbers, so every headline figure gets
5
+ an interval.
6
+
7
+ How it works: test accounts are drawn with replacement, as many as there are, and every result
8
+ is recomputed on the new sample; this is repeated (1,000 times by default) and the 2.5th and
9
+ 97.5th percentiles of the repeats form the 95% interval (the percentile bootstrap, Efron &
10
+ Tibshirani 1993, ch. 13). Rows with identical inputs are drawn together, matching the
11
+ group-aware split. Comparisons (model minus rule, one model minus another) are computed on the
12
+ same draw, so the interval of a difference accounts for the two results moving together.
13
+
14
+ What is held fixed: the trained models, the threshold, the calibration map and the chosen
15
+ rule. The intervals therefore measure how much the test figures depend on which customers
16
+ happened to be in the test sample. They do not include the variation from retraining on
17
+ different data or with another seed, which would widen them.
18
+
19
+ Instead of copying rows, each draw is a vector of weights: how many times each row was
20
+ drawn (0, 1, 2, ...). Weight 1 for every row reproduces the original figures exactly.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ from collections.abc import Callable
26
+
27
+ import numpy as np
28
+ import pandas as pd
29
+
30
+ DEFAULT_REPLICATES = 1000
31
+ LEVEL = 0.95
32
+
33
+
34
+ def resample_weights(groups, n_boot: int, seed: int = 0):
35
+ """Yield ``n_boot`` weight vectors, one per bootstrap sample.
36
+
37
+ ``groups`` gives each row's group id; groups are drawn with replacement, as many as there
38
+ are, and every row of a group gets the group's draw count.
39
+ """
40
+ g = pd.factorize(np.asarray(groups))[0]
41
+ n_groups = int(g.max()) + 1 if len(g) else 0
42
+ rng = np.random.default_rng(seed)
43
+ for _ in range(n_boot):
44
+ counts = np.bincount(rng.integers(0, n_groups, n_groups), minlength=n_groups)
45
+ yield counts[g].astype(float)
46
+
47
+
48
+ def capture_rate(w, y, flag) -> float:
49
+ """Weighted share of positives that are flagged."""
50
+ pos = w * (np.asarray(y) == 1)
51
+ total = pos.sum()
52
+ return float((pos * flag).sum() / total) if total else float("nan")
53
+
54
+
55
+ def precision(w, y, flag) -> float:
56
+ """Weighted share of flagged rows that are positive."""
57
+ f = w * np.asarray(flag, bool)
58
+ total = f.sum()
59
+ return float((f * (np.asarray(y) == 1)).sum() / total) if total else float("nan")
60
+
61
+
62
+ def flagged_fraction(w, flag) -> float:
63
+ return float((w * np.asarray(flag, bool)).sum() / w.sum())
64
+
65
+
66
+ class RankedScores:
67
+ """A score vector prepared once for fast weighted ROC AUC on many bootstrap samples."""
68
+
69
+ def __init__(self, y, score):
70
+ self.pos = np.asarray(y) == 1
71
+ _, self.inverse = np.unique(np.asarray(score, float), return_inverse=True)
72
+ self.n_unique = int(self.inverse.max()) + 1 if len(self.inverse) else 0
73
+
74
+ def auc(self, w) -> float:
75
+ """Weighted ROC AUC: the probability that a random positive outscores a random
76
+ negative, ties counting one half (the Mann-Whitney form)."""
77
+ pos = np.bincount(self.inverse, weights=w * self.pos, minlength=self.n_unique)
78
+ neg = np.bincount(self.inverse, weights=w * ~self.pos, minlength=self.n_unique)
79
+ below = np.cumsum(neg) - neg
80
+ denom = pos.sum() * neg.sum()
81
+ return float((pos * (below + 0.5 * neg)).sum() / denom) if denom else float("nan")
82
+
83
+
84
+ def weighted_roc_auc(y, score, w=None) -> float:
85
+ """ROC AUC with row weights (equal to scikit-learn's roc_auc_score with sample_weight)."""
86
+ w = np.ones(len(y)) if w is None else np.asarray(w, float)
87
+ return RankedScores(y, score).auc(w)
88
+
89
+
90
+ def cost(w, y, flag, cost_fp: float, missed) -> float:
91
+ """Weighted cost of a rule: cost_fp per flagged negative plus the missed cost of every
92
+ positive that is not flagged."""
93
+ y, flag = np.asarray(y) == 1, np.asarray(flag, bool)
94
+ return float(cost_fp * (w * (flag & ~y)).sum() + (w * missed * (~flag & y)).sum())
95
+
96
+
97
+ def saving_vs_best_trivial(w, y, flag, cost_fp: float, missed) -> float:
98
+ """Cost saving against the better of flagging nobody and flagging everybody, as a
99
+ fraction of that strategy's cost (same definition as decision.summarize)."""
100
+ none = cost(w, y, np.zeros(len(y), bool), cost_fp, missed)
101
+ everyone = cost(w, y, np.ones(len(y), bool), cost_fp, missed)
102
+ best = min(none, everyone)
103
+ return float((best - cost(w, y, flag, cost_fp, missed)) / best) if best else 0.0
104
+
105
+
106
+ def bootstrap(statistic: Callable[[np.ndarray], dict], groups, n_boot: int = DEFAULT_REPLICATES,
107
+ seed: int = 0, level: float = LEVEL, differences=(),
108
+ units: dict | None = None) -> pd.DataFrame:
109
+ """Percentile bootstrap intervals for every number ``statistic`` returns.
110
+
111
+ ``statistic(w)`` takes a vector of row weights and returns ``{name: value}``. Names listed
112
+ in ``differences`` are comparisons; for those the table says whether the interval excludes
113
+ zero, i.e. whether the sign of the difference is clear at this confidence level. ``units``
114
+ maps names to "fraction", "relative", "auc" or "ratio" (used only for printing; default
115
+ "fraction").
116
+ """
117
+ units = units or {}
118
+ if n_boot < 1:
119
+ raise ValueError("n_boot must be at least 1")
120
+ if not 0 < level < 1:
121
+ raise ValueError("level must be between 0 and 1")
122
+ n = len(np.asarray(groups))
123
+ estimate = statistic(np.ones(n))
124
+ draws = {k: np.empty(n_boot) for k in estimate}
125
+ for b, w in enumerate(resample_weights(groups, n_boot, seed)):
126
+ for k, v in statistic(w).items():
127
+ draws[k][b] = v
128
+ tail = (1 - level) / 2
129
+ rows = []
130
+ for k, v in estimate.items():
131
+ d = draws[k]
132
+ lo, hi = np.nanquantile(d, [tail, 1 - tail])
133
+ is_diff = k in differences
134
+ rows.append({"quantity": k, "type": "difference" if is_diff else "estimate",
135
+ "unit": units.get(k, "fraction"),
136
+ "estimate": float(v), "ci_low": float(lo), "ci_high": float(hi),
137
+ "std_error": float(np.nanstd(d, ddof=1)) if n_boot > 1 else float("nan"),
138
+ "excludes_zero": bool(lo > 0 or hi < 0) if is_diff else None})
139
+ out = pd.DataFrame(rows)
140
+ out.attrs.update({"replicates": n_boot, "level": level, "seed": seed})
141
+ return out
142
+
143
+
144
+ def format_interval(row) -> str:
145
+ """One line per quantity: estimate and interval in readable units. A difference of two
146
+ fractions is in percentage points; a relative error (unit "relative") in percent."""
147
+ unit, diff = row["unit"], row["type"] == "difference"
148
+ sign = "+" if diff else ""
149
+ if unit in ("fraction", "relative"):
150
+ suffix = " points" if diff and unit == "fraction" else "%"
151
+ est, lo, hi = (f"{100 * row[c]:{sign}.1f}" for c in ("estimate", "ci_low", "ci_high"))
152
+ text = (f"{est}{suffix} [{lo}, {hi}]" if suffix == " points"
153
+ else f"{est}% [{lo}%, {hi}%]")
154
+ else:
155
+ digits = 3 if unit == "auc" else 2
156
+ est, lo, hi = (f"{row[c]:{sign}.{digits}f}" for c in ("estimate", "ci_low", "ci_high"))
157
+ text = f"{est} [{lo}, {hi}]"
158
+ if diff:
159
+ text += "; sign clear" if row["excludes_zero"] else "; includes zero"
160
+ return f"{row['quantity']}: {text}"
@@ -0,0 +1,128 @@
1
+ """Probability calibration.
2
+
3
+ A model can rank accounts well and still give probabilities that are too high or too low.
4
+ Ranking is enough to decide *which* accounts to act on; calibrated probabilities are needed to
5
+ answer *how many* defaults to expect (staffing, provisioning: expected loss = PD x LGD x
6
+ exposure) and to make per-account expected-cost decisions (creditdecide.decision).
7
+
8
+ Method
9
+ ------
10
+ The scoring model is fixed. A one-dimensional map from its raw score to a probability is
11
+ fitted on the VALIDATION split, which the model never trained on:
12
+
13
+ * ``none`` identity;
14
+ * ``sigmoid`` logistic regression on the log-odds of the raw score (Platt scaling);
15
+ * ``isotonic`` monotone step function (pool-adjacent-violators).
16
+
17
+ The method is chosen by 5-fold stratified cross-validation *within* the validation split on log
18
+ loss (a proper scoring rule), with the one-standard-error rule: the simplest method (none, then
19
+ sigmoid, then isotonic) whose cross-validated log loss is within one standard error of the best
20
+ is used, so a map is only added when it clearly helps. It is then refitted on the whole
21
+ validation split. The test split is only used to report the result. Every map is monotone
22
+ non-decreasing, so the ranking of accounts, and therefore every threshold decision, is
23
+ unchanged by calibration.
24
+
25
+ The fitted map is stored as plain numbers (two coefficients, or the isotonic breakpoints) in the
26
+ model manifest and applied with ``np.interp``, so no extra code needs to be deserialized.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import numpy as np
32
+ import pandas as pd
33
+ from sklearn.isotonic import IsotonicRegression
34
+ from sklearn.linear_model import LogisticRegression
35
+ from sklearn.metrics import brier_score_loss, log_loss
36
+ from sklearn.model_selection import StratifiedKFold
37
+
38
+ METHODS = ("none", "sigmoid", "isotonic")
39
+ EPS = 1e-6
40
+ N_BINS = 10
41
+
42
+
43
+ def _logit(p):
44
+ p = np.clip(np.asarray(p, float), EPS, 1 - EPS)
45
+ return np.log(p / (1 - p))
46
+
47
+
48
+ def fit_map(method: str, score, y) -> dict:
49
+ """Fit one calibration map; returns its JSON-serialisable parameters."""
50
+ score, y = np.asarray(score, float), np.asarray(y, int)
51
+ if method == "none":
52
+ return {"method": "none"}
53
+ if method == "sigmoid":
54
+ lr = LogisticRegression(C=1e6, max_iter=1000).fit(_logit(score)[:, None], y)
55
+ return {"method": "sigmoid", "a": float(lr.coef_[0, 0]), "b": float(lr.intercept_[0])}
56
+ if method == "isotonic":
57
+ iso = IsotonicRegression(out_of_bounds="clip", y_min=0.0, y_max=1.0).fit(score, y)
58
+ return {"method": "isotonic", "x": iso.X_thresholds_.tolist(),
59
+ "y": iso.y_thresholds_.tolist()}
60
+ raise ValueError(f"unknown calibration method {method!r}; choose from {METHODS}")
61
+
62
+
63
+ def apply_map(params: dict, score) -> np.ndarray:
64
+ score = np.asarray(score, float)
65
+ m = params["method"]
66
+ if m == "none":
67
+ return np.clip(score, 0.0, 1.0)
68
+ if m == "sigmoid":
69
+ return 1.0 / (1.0 + np.exp(-(params["a"] * _logit(score) + params["b"])))
70
+ if m == "isotonic":
71
+ return np.interp(score, params["x"], params["y"]) # constant beyond the end points
72
+ raise ValueError(f"unknown calibration method {m!r}")
73
+
74
+
75
+ def choose_and_fit(score_val, y_val, method: str = "auto", seed: int = 0,
76
+ n_folds: int = 5) -> tuple[dict, pd.DataFrame]:
77
+ """Pick a method by cross-validated log loss on the validation split (or use ``method``),
78
+ then fit it on the whole validation split. Returns (params, cross-validation table)."""
79
+ s, y = np.asarray(score_val, float), np.asarray(y_val, int)
80
+ rows = []
81
+ folds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)
82
+ for m in METHODS:
83
+ ll, br = [], []
84
+ for fit_idx, ev_idx in folds.split(s, y):
85
+ p = apply_map(fit_map(m, s[fit_idx], y[fit_idx]), s[ev_idx])
86
+ ll.append(log_loss(y[ev_idx], np.clip(p, EPS, 1 - EPS), labels=[0, 1]))
87
+ br.append(brier_score_loss(y[ev_idx], p))
88
+ rows.append({"method": m, "cv_log_loss": float(np.mean(ll)),
89
+ "cv_log_loss_se": float(np.std(ll, ddof=1) / np.sqrt(n_folds)),
90
+ "cv_brier": float(np.mean(br))})
91
+ table = pd.DataFrame(rows)
92
+ if method == "auto": # one-standard-error rule; METHODS is ordered simplest first
93
+ best = table.loc[table.cv_log_loss.idxmin()]
94
+ ok = table.cv_log_loss <= best.cv_log_loss + best.cv_log_loss_se
95
+ method = str(table.loc[ok, "method"].iloc[0])
96
+ elif method not in METHODS:
97
+ raise ValueError(f"calibration must be 'auto' or one of {METHODS}")
98
+ table["chosen"] = table["method"] == method
99
+ return fit_map(method, s, y), table
100
+
101
+
102
+ def reliability_table(y, p, n_bins: int = N_BINS) -> pd.DataFrame:
103
+ """Equal-count bins of predicted probability: mean prediction vs observed rate."""
104
+ y, p = np.asarray(y, int), np.asarray(p, float)
105
+ order = np.argsort(p, kind="stable")
106
+ rows = []
107
+ for i, idx in enumerate(np.array_split(order, n_bins), start=1):
108
+ if len(idx) == 0:
109
+ continue
110
+ rows.append({"bin": i, "n": len(idx), "mean_predicted": float(p[idx].mean()),
111
+ "observed_rate": float(y[idx].mean())})
112
+ return pd.DataFrame(rows)
113
+
114
+
115
+ def calibration_metrics(y, p, n_bins: int = N_BINS) -> dict:
116
+ """Brier score, log loss, expected calibration error (equal-count bins), and the
117
+ portfolio check: expected number of positives (sum of probabilities) vs actual."""
118
+ y, p = np.asarray(y, int), np.asarray(p, float)
119
+ t = reliability_table(y, p, n_bins)
120
+ ece = float(np.sum(t["n"] * (t["mean_predicted"] - t["observed_rate"]).abs()) / len(y))
121
+ return {
122
+ "brier": float(brier_score_loss(y, p)),
123
+ "log_loss": float(log_loss(y, np.clip(p, EPS, 1 - EPS), labels=[0, 1])),
124
+ "ece": ece,
125
+ "max_bin_gap": float((t["mean_predicted"] - t["observed_rate"]).abs().max()),
126
+ "expected_positives": float(p.sum()),
127
+ "actual_positives": int(y.sum()),
128
+ }