quantology-modelmaker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelmaker/__init__.py +2 -0
- modelmaker/api.py +1073 -0
- modelmaker/blocks/__init__.py +0 -0
- modelmaker/blocks/base.py +78 -0
- modelmaker/blocks/library.py +327 -0
- modelmaker/blocks/modelling.py +312 -0
- modelmaker/blocks/stat_tests.py +133 -0
- modelmaker/cache.py +44 -0
- modelmaker/compiler.py +478 -0
- modelmaker/graph.py +171 -0
- modelmaker/llm/__init__.py +28 -0
- modelmaker/llm/anthropic_provider.py +79 -0
- modelmaker/llm/base.py +85 -0
- modelmaker/llm/claude_cli_provider.py +90 -0
- modelmaker/llm/gemini_provider.py +138 -0
- modelmaker/llm/lmstudio_provider.py +140 -0
- modelmaker/llm/openai_provider.py +138 -0
- modelmaker/llm/prompts.py +189 -0
- modelmaker/llm/settings.py +47 -0
- modelmaker/llm/stub_provider.py +29 -0
- modelmaker/metadata_transforms.py +109 -0
- modelmaker/packet.py +111 -0
- modelmaker/project.py +129 -0
- modelmaker/run_worker.py +143 -0
- modelmaker/runner.py +1313 -0
- modelmaker/session.py +400 -0
- modelmaker/static/assets/index-B9cHjTMW.css +1 -0
- modelmaker/static/assets/index-EvjTc-1k.js +34 -0
- modelmaker/static/brand-quantology/favicon.svg +4 -0
- modelmaker/static/brand-quantology/icon-mark-light.svg +13 -0
- modelmaker/static/icons.svg +24 -0
- modelmaker/static/index.html +20 -0
- modelmaker/tui/__init__.py +0 -0
- modelmaker/tui/app.py +785 -0
- modelmaker/tui/canvas.py +388 -0
- modelmaker/tui/charts.py +84 -0
- modelmaker/tui/client.py +270 -0
- modelmaker/tui/inspector.py +285 -0
- modelmaker/tui/paramspecs.py +128 -0
- modelmaker/tui/preview.py +71 -0
- modelmaker/tui/screens.py +182 -0
- modelmaker/util.py +44 -0
- quantology_modelmaker-0.1.0.dist-info/METADATA +247 -0
- quantology_modelmaker-0.1.0.dist-info/RECORD +49 -0
- quantology_modelmaker-0.1.0.dist-info/WHEEL +5 -0
- quantology_modelmaker-0.1.0.dist-info/entry_points.txt +3 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/LICENSE +204 -0
- quantology_modelmaker-0.1.0.dist-info/licenses/NOTICE +12 -0
- quantology_modelmaker-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Modelling block library: GLM, logistic regression, and weight-of-evidence
|
|
2
|
+
(WoE) transform -- the core toolkit for a credit-risk / PD-style workflow.
|
|
3
|
+
Same contract as blocks/library.py: each `fn` is a plain function over
|
|
4
|
+
pl.DataFrame / literal params, no DataFramePacket. A "model" output port
|
|
5
|
+
(see blocks/base.py PortType) carries a plain, JSON-shaped dict describing
|
|
6
|
+
the fitted model -- not a pickled estimator -- so it stays inspectable and
|
|
7
|
+
never needs a special deserializer downstream.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import polars as pl
|
|
13
|
+
|
|
14
|
+
from ..packet import ColumnMeta, ColumnRole
|
|
15
|
+
from .base import BlockSpec, PortSpec, register_block
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _predictions_meta(new_col_roles: dict[str, ColumnRole]):
|
|
19
|
+
"""Tag each of this block's own new output columns with the given role;
|
|
20
|
+
every other output column is just the input passed through unchanged.
|
|
21
|
+
`predicted` is a unique role (see packet.UNIQUE_ROLES) -- a block that
|
|
22
|
+
emits more than one prediction-shaped column (logistic_regression's
|
|
23
|
+
predicted_proba/predicted_class) can only give PREDICTED to the one
|
|
24
|
+
that's actually the model's score; the rest fall back to FEATURE, same
|
|
25
|
+
as any other derived column."""
|
|
26
|
+
|
|
27
|
+
def _fn(input_metas, outputs, params):
|
|
28
|
+
(in_meta,) = input_metas.values()
|
|
29
|
+
(df,) = outputs.values()
|
|
30
|
+
result = dict(in_meta)
|
|
31
|
+
for name, role in new_col_roles.items():
|
|
32
|
+
result[name] = ColumnMeta(dtype=str(df.schema[name]), role=role)
|
|
33
|
+
return {"predictions": {name: result[name] for name in df.columns if name in result}}
|
|
34
|
+
|
|
35
|
+
return _fn
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def glm_fit(
|
|
39
|
+
df: pl.DataFrame,
|
|
40
|
+
target: str,
|
|
41
|
+
features: list[str],
|
|
42
|
+
family: str = "gaussian",
|
|
43
|
+
alpha: float = 0.0,
|
|
44
|
+
) -> tuple[pl.DataFrame, dict]:
|
|
45
|
+
from sklearn.linear_model import TweedieRegressor
|
|
46
|
+
|
|
47
|
+
power = {"gaussian": 0.0, "poisson": 1.0, "gamma": 2.0, "inverse_gaussian": 3.0}.get(family)
|
|
48
|
+
if power is None:
|
|
49
|
+
raise ValueError(f"unknown GLM family: {family!r} (use gaussian, poisson, gamma, or inverse_gaussian)")
|
|
50
|
+
|
|
51
|
+
x = df.select(features).to_numpy()
|
|
52
|
+
y = df[target].to_numpy()
|
|
53
|
+
model = TweedieRegressor(power=power, alpha=alpha, link="auto", max_iter=500)
|
|
54
|
+
model.fit(x, y)
|
|
55
|
+
|
|
56
|
+
predictions = df.with_columns(pl.Series("predicted", model.predict(x)))
|
|
57
|
+
artifact = {
|
|
58
|
+
"kind": "glm",
|
|
59
|
+
"family": family,
|
|
60
|
+
"target": target,
|
|
61
|
+
"features": features,
|
|
62
|
+
"coefficients": dict(zip(features, [float(c) for c in model.coef_])),
|
|
63
|
+
"intercept": float(model.intercept_),
|
|
64
|
+
"alpha": alpha,
|
|
65
|
+
}
|
|
66
|
+
return predictions, artifact
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
register_block(
|
|
70
|
+
BlockSpec(
|
|
71
|
+
category="glm_fit",
|
|
72
|
+
block_type="standard",
|
|
73
|
+
group="modelling",
|
|
74
|
+
display_name="GLM fit",
|
|
75
|
+
inputs=[PortSpec("df")],
|
|
76
|
+
outputs=[PortSpec("predictions"), PortSpec("model", type="model", required=False)],
|
|
77
|
+
fn=glm_fit,
|
|
78
|
+
metadata_transform=_predictions_meta({"predicted": ColumnRole.PREDICTED}),
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def logistic_regression(
|
|
84
|
+
df: pl.DataFrame,
|
|
85
|
+
target: str,
|
|
86
|
+
features: list[str],
|
|
87
|
+
C: float = 1.0,
|
|
88
|
+
max_iter: int = 200,
|
|
89
|
+
) -> tuple[pl.DataFrame, dict]:
|
|
90
|
+
from sklearn.linear_model import LogisticRegression
|
|
91
|
+
|
|
92
|
+
x = df.select(features).to_numpy()
|
|
93
|
+
y = df[target].to_numpy()
|
|
94
|
+
model = LogisticRegression(C=C, max_iter=max_iter)
|
|
95
|
+
model.fit(x, y)
|
|
96
|
+
|
|
97
|
+
proba = model.predict_proba(x)[:, 1]
|
|
98
|
+
predictions = df.with_columns(
|
|
99
|
+
pl.Series("predicted_proba", proba),
|
|
100
|
+
pl.Series("predicted_class", model.predict(x)),
|
|
101
|
+
)
|
|
102
|
+
artifact = {
|
|
103
|
+
"kind": "logistic_regression",
|
|
104
|
+
"target": target,
|
|
105
|
+
"features": features,
|
|
106
|
+
"coefficients": dict(zip(features, [float(c) for c in model.coef_[0]])),
|
|
107
|
+
"intercept": float(model.intercept_[0]),
|
|
108
|
+
"C": C,
|
|
109
|
+
}
|
|
110
|
+
return predictions, artifact
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
register_block(
|
|
114
|
+
BlockSpec(
|
|
115
|
+
category="logistic_regression",
|
|
116
|
+
block_type="standard",
|
|
117
|
+
group="modelling",
|
|
118
|
+
display_name="Logistic regression",
|
|
119
|
+
inputs=[PortSpec("df")],
|
|
120
|
+
outputs=[PortSpec("predictions"), PortSpec("model", type="model", required=False)],
|
|
121
|
+
fn=logistic_regression,
|
|
122
|
+
metadata_transform=_predictions_meta(
|
|
123
|
+
{"predicted_proba": ColumnRole.PREDICTED, "predicted_class": ColumnRole.FEATURE}
|
|
124
|
+
),
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def woe_transform(df: pl.DataFrame, col: str, target: str, bins: int = 10) -> pl.DataFrame:
|
|
130
|
+
"""Weight-of-evidence encoding of `col` against a binary `target`
|
|
131
|
+
(1 = event/bad, 0 = non-event/good), the standard credit-scoring
|
|
132
|
+
transform: WoE = ln(% of goods in bucket / % of bads in bucket)."""
|
|
133
|
+
out_col = f"{col}_woe"
|
|
134
|
+
is_numeric = df.schema[col].is_numeric()
|
|
135
|
+
bucket_expr = (
|
|
136
|
+
pl.col(col).qcut(bins, allow_duplicates=True).alias("_bucket") if is_numeric else pl.col(col).cast(pl.Utf8).alias("_bucket")
|
|
137
|
+
)
|
|
138
|
+
bucketed = df.with_columns(bucket_expr)
|
|
139
|
+
tagged = bucketed.select("_bucket", pl.col(target).alias("_target"))
|
|
140
|
+
|
|
141
|
+
total_goods = (tagged["_target"] == 0).sum()
|
|
142
|
+
total_bads = (tagged["_target"] == 1).sum()
|
|
143
|
+
# Laplace-style smoothing (+0.5) so an all-good or all-bad bucket never
|
|
144
|
+
# produces ln(0) / a divide-by-zero.
|
|
145
|
+
stats = tagged.group_by("_bucket").agg(
|
|
146
|
+
goods=(pl.col("_target") == 0).sum(),
|
|
147
|
+
bads=(pl.col("_target") == 1).sum(),
|
|
148
|
+
)
|
|
149
|
+
stats = stats.with_columns(
|
|
150
|
+
(((pl.col("goods") + 0.5) / (total_goods + 0.5)) / ((pl.col("bads") + 0.5) / (total_bads + 0.5))).log().alias("_woe")
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
result = bucketed.join(stats.select("_bucket", "_woe"), on="_bucket", how="left")
|
|
154
|
+
result = result.rename({"_woe": out_col}).drop("_bucket")
|
|
155
|
+
return result
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _woe_meta(input_metas, outputs, params):
|
|
159
|
+
(in_meta,) = input_metas.values()
|
|
160
|
+
(df,) = outputs.values()
|
|
161
|
+
out_col = f"{params['col']}_woe"
|
|
162
|
+
result = dict(in_meta)
|
|
163
|
+
result[out_col] = ColumnMeta(dtype=str(df.schema[out_col]), role=ColumnRole.FEATURE)
|
|
164
|
+
return {"out": {name: result[name] for name in df.columns if name in result}}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def fit_master_scale(
|
|
168
|
+
df: pl.DataFrame,
|
|
169
|
+
score_col: str,
|
|
170
|
+
target_col: str,
|
|
171
|
+
n_grades: int = 10,
|
|
172
|
+
algorithm: str = "quantile",
|
|
173
|
+
) -> dict:
|
|
174
|
+
"""Derives rating-grade cutoffs over `score_col` -- fit once (e.g. on a
|
|
175
|
+
development sample) and reused by assign_rating_grade on every later
|
|
176
|
+
run, so grade boundaries stay stable rather than shifting with whatever
|
|
177
|
+
data happens to be on hand the way a plain per-run qcut would. Grade 1
|
|
178
|
+
is the lowest score/risk, grade `n_grades` the highest (or fewer, for
|
|
179
|
+
'monotonic_default_rate' -- see below).
|
|
180
|
+
|
|
181
|
+
`algorithm`:
|
|
182
|
+
- "quantile": equal population per grade.
|
|
183
|
+
- "equal_width": equal-width bins between the observed min and max.
|
|
184
|
+
- "monotonic_default_rate": pool-adjacent-violators over fine
|
|
185
|
+
quantile bins, merging any pair whose observed default rate would
|
|
186
|
+
otherwise fall from one grade to the next -- a real rating scale's
|
|
187
|
+
default rate must rise monotonically with risk. Can return fewer
|
|
188
|
+
than `n_grades` grades when the data doesn't support that many
|
|
189
|
+
without violating monotonicity; never more.
|
|
190
|
+
"""
|
|
191
|
+
import numpy as np
|
|
192
|
+
|
|
193
|
+
scores = df[score_col].to_numpy().astype(float)
|
|
194
|
+
targets = df[target_col].to_numpy().astype(float)
|
|
195
|
+
if np.unique(scores).size < 2:
|
|
196
|
+
raise ValueError(f"'{score_col}' has fewer than 2 distinct values -- cannot fit a master scale")
|
|
197
|
+
if n_grades < 1:
|
|
198
|
+
raise ValueError("n_grades must be at least 1")
|
|
199
|
+
|
|
200
|
+
if algorithm == "quantile":
|
|
201
|
+
edges = np.unique(np.quantile(scores, np.linspace(0, 1, n_grades + 1)))
|
|
202
|
+
elif algorithm == "equal_width":
|
|
203
|
+
edges = np.unique(np.linspace(scores.min(), scores.max(), n_grades + 1))
|
|
204
|
+
elif algorithm == "monotonic_default_rate":
|
|
205
|
+
|
|
206
|
+
def _pava_edges() -> np.ndarray:
|
|
207
|
+
n_fine = min(max(n_grades * 5, 50), np.unique(scores).size)
|
|
208
|
+
fine_edges = np.unique(np.quantile(scores, np.linspace(0, 1, n_fine + 1)))
|
|
209
|
+
bin_idx = np.clip(np.searchsorted(fine_edges, scores, side="right") - 1, 0, len(fine_edges) - 2)
|
|
210
|
+
|
|
211
|
+
# Each pool: [sum(target), count, first fine-bin, last fine-bin].
|
|
212
|
+
pools: list[list[float]] = []
|
|
213
|
+
for b in range(len(fine_edges) - 1):
|
|
214
|
+
mask = bin_idx == b
|
|
215
|
+
if mask.any():
|
|
216
|
+
pools.append([float(targets[mask].sum()), int(mask.sum()), b, b])
|
|
217
|
+
|
|
218
|
+
# Pool-adjacent-violators: merge a pool into its predecessor
|
|
219
|
+
# whenever its default rate would otherwise be lower.
|
|
220
|
+
stack: list[list[float]] = []
|
|
221
|
+
for pool in pools:
|
|
222
|
+
stack.append(pool)
|
|
223
|
+
while len(stack) > 1 and (stack[-2][0] / stack[-2][1]) > (stack[-1][0] / stack[-1][1]):
|
|
224
|
+
b_pool = stack.pop()
|
|
225
|
+
a_pool = stack.pop()
|
|
226
|
+
stack.append([a_pool[0] + b_pool[0], a_pool[1] + b_pool[1], a_pool[2], b_pool[3]])
|
|
227
|
+
|
|
228
|
+
# Merge down to n_grades if PAVA still left more pools than
|
|
229
|
+
# requested, always combining whichever adjacent pair has the
|
|
230
|
+
# closest default rate -- never split a pool further apart.
|
|
231
|
+
while len(stack) > n_grades:
|
|
232
|
+
rates = [p[0] / p[1] for p in stack]
|
|
233
|
+
gaps = [rates[i + 1] - rates[i] for i in range(len(stack) - 1)]
|
|
234
|
+
i = min(range(len(gaps)), key=lambda j: gaps[j])
|
|
235
|
+
a_pool, b_pool = stack[i], stack[i + 1]
|
|
236
|
+
stack[i : i + 2] = [[a_pool[0] + b_pool[0], a_pool[1] + b_pool[1], a_pool[2], b_pool[3]]]
|
|
237
|
+
|
|
238
|
+
return np.array([fine_edges[0]] + [fine_edges[p[3] + 1] for p in stack], dtype=float)
|
|
239
|
+
|
|
240
|
+
edges = _pava_edges()
|
|
241
|
+
else:
|
|
242
|
+
raise ValueError(
|
|
243
|
+
f"unknown algorithm {algorithm!r} (expected 'quantile', 'equal_width', or 'monotonic_default_rate')"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
edges = edges.astype(float)
|
|
247
|
+
edges[0], edges[-1] = -np.inf, np.inf
|
|
248
|
+
grades = [{"grade": i + 1, "lower": float(edges[i]), "upper": float(edges[i + 1])} for i in range(len(edges) - 1)]
|
|
249
|
+
|
|
250
|
+
return {"kind": "master_scale", "score_col": score_col, "algorithm": algorithm, "grades": grades}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
register_block(
|
|
254
|
+
BlockSpec(
|
|
255
|
+
category="fit_master_scale",
|
|
256
|
+
block_type="standard",
|
|
257
|
+
group="modelling",
|
|
258
|
+
display_name="Fit master scale",
|
|
259
|
+
inputs=[PortSpec("df")],
|
|
260
|
+
outputs=[PortSpec("master_scale", type="master_scale")],
|
|
261
|
+
fn=fit_master_scale,
|
|
262
|
+
metadata_transform=lambda *_a, **_k: {},
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def assign_rating_grade(df: pl.DataFrame, master_scale: dict, score_col: str | None = None) -> pl.DataFrame:
|
|
268
|
+
"""Buckets `score_col` (defaults to whichever column the master scale
|
|
269
|
+
was fit on) into the grades `master_scale` describes -- the apply half
|
|
270
|
+
of fit_master_scale, so the same cutoffs are reused run after run
|
|
271
|
+
instead of being re-derived from whatever data happens to be on hand."""
|
|
272
|
+
col = score_col or master_scale["score_col"]
|
|
273
|
+
grades = master_scale["grades"]
|
|
274
|
+
breaks = [g["upper"] for g in grades[:-1]]
|
|
275
|
+
labels = [str(g["grade"]) for g in grades]
|
|
276
|
+
return df.with_columns(pl.col(col).cut(breaks, labels=labels).cast(pl.Utf8).alias("grade"))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _rating_grade_meta(input_metas, outputs, params):
|
|
280
|
+
(in_meta,) = input_metas.values()
|
|
281
|
+
(df,) = outputs.values()
|
|
282
|
+
result = dict(in_meta)
|
|
283
|
+
result["grade"] = ColumnMeta(dtype=str(df.schema["grade"]), role=ColumnRole.SEGMENT)
|
|
284
|
+
return {"out": {name: result[name] for name in df.columns if name in result}}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
register_block(
|
|
288
|
+
BlockSpec(
|
|
289
|
+
category="assign_rating_grade",
|
|
290
|
+
block_type="standard",
|
|
291
|
+
group="modelling",
|
|
292
|
+
display_name="Assign rating grade",
|
|
293
|
+
inputs=[PortSpec("df"), PortSpec("master_scale", type="master_scale")],
|
|
294
|
+
outputs=[PortSpec("out")],
|
|
295
|
+
fn=assign_rating_grade,
|
|
296
|
+
metadata_transform=_rating_grade_meta,
|
|
297
|
+
)
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
register_block(
|
|
302
|
+
BlockSpec(
|
|
303
|
+
category="woe_transform",
|
|
304
|
+
block_type="standard",
|
|
305
|
+
group="modelling",
|
|
306
|
+
display_name="Weight of evidence",
|
|
307
|
+
inputs=[PortSpec("df")],
|
|
308
|
+
outputs=[PortSpec("out")],
|
|
309
|
+
fn=woe_transform,
|
|
310
|
+
metadata_transform=_woe_meta,
|
|
311
|
+
)
|
|
312
|
+
)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Model-validation test blocks: KS statistic, AUC/Gini, and Population
|
|
2
|
+
Stability Index (PSI) -- the standard discrimination and stability checks
|
|
3
|
+
for a PD-style model. Each produces a scalar_metric artifact (a plain
|
|
4
|
+
dict), not a dataframe, so it terminates a branch of the pipeline the way
|
|
5
|
+
generate_image/display_table do. Extra libraries (scipy, sklearn, numpy)
|
|
6
|
+
are imported locally inside each fn -- see generate_image in library.py for
|
|
7
|
+
why: the function body is inlined verbatim into the compiled script, which
|
|
8
|
+
only guarantees `polars as pl` at module level.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import polars as pl
|
|
14
|
+
|
|
15
|
+
from .base import BlockSpec, PortSpec, register_block
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def ks_test(df: pl.DataFrame, score_col: str, target_col: str) -> dict:
|
|
19
|
+
from scipy.stats import ks_2samp
|
|
20
|
+
|
|
21
|
+
scores = df[score_col].to_numpy()
|
|
22
|
+
target = df[target_col].to_numpy()
|
|
23
|
+
good = scores[target == 0]
|
|
24
|
+
bad = scores[target == 1]
|
|
25
|
+
if len(good) == 0 or len(bad) == 0:
|
|
26
|
+
raise ValueError(f"'{target_col}' must contain both 0 and 1 values to run a KS test")
|
|
27
|
+
result = ks_2samp(good, bad)
|
|
28
|
+
return {"kind": "ks_test", "ks_statistic": float(result.statistic), "p_value": float(result.pvalue)}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
register_block(
|
|
32
|
+
BlockSpec(
|
|
33
|
+
category="ks_test",
|
|
34
|
+
block_type="output",
|
|
35
|
+
group="tests",
|
|
36
|
+
display_name="KS test",
|
|
37
|
+
inputs=[PortSpec("df")],
|
|
38
|
+
outputs=[PortSpec("metric", type="scalar_metric")],
|
|
39
|
+
fn=ks_test,
|
|
40
|
+
metadata_transform=lambda *_a, **_k: {},
|
|
41
|
+
)
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def auc_gini(df: pl.DataFrame, score_col: str, target_col: str) -> dict:
|
|
46
|
+
from sklearn.metrics import roc_auc_score
|
|
47
|
+
|
|
48
|
+
auc = float(roc_auc_score(df[target_col].to_numpy(), df[score_col].to_numpy()))
|
|
49
|
+
return {"kind": "auc_gini", "auc": auc, "gini": 2 * auc - 1}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
register_block(
|
|
53
|
+
BlockSpec(
|
|
54
|
+
category="auc_gini",
|
|
55
|
+
block_type="output",
|
|
56
|
+
group="tests",
|
|
57
|
+
display_name="AUC / Gini",
|
|
58
|
+
inputs=[PortSpec("df")],
|
|
59
|
+
outputs=[PortSpec("metric", type="scalar_metric")],
|
|
60
|
+
fn=auc_gini,
|
|
61
|
+
metadata_transform=lambda *_a, **_k: {},
|
|
62
|
+
)
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def psi_test(expected: pl.DataFrame, actual: pl.DataFrame, col: str, bins: int = 10) -> dict:
|
|
67
|
+
"""Population Stability Index: how much `col`'s distribution in `actual`
|
|
68
|
+
has drifted from `expected` (e.g. current score distribution vs. the
|
|
69
|
+
distribution the model was built/validated on)."""
|
|
70
|
+
import numpy as np
|
|
71
|
+
|
|
72
|
+
exp_vals = expected[col].to_numpy()
|
|
73
|
+
act_vals = actual[col].to_numpy()
|
|
74
|
+
edges = np.quantile(exp_vals, np.linspace(0, 1, bins + 1))
|
|
75
|
+
edges[0], edges[-1] = -np.inf, np.inf
|
|
76
|
+
edges = np.unique(edges)
|
|
77
|
+
|
|
78
|
+
exp_counts, _ = np.histogram(exp_vals, bins=edges)
|
|
79
|
+
act_counts, _ = np.histogram(act_vals, bins=edges)
|
|
80
|
+
exp_pct = exp_counts / max(len(exp_vals), 1)
|
|
81
|
+
act_pct = act_counts / max(len(act_vals), 1)
|
|
82
|
+
# Laplace-style smoothing so an empty bucket never produces ln(0).
|
|
83
|
+
eps = 1e-6
|
|
84
|
+
contributions = (act_pct - exp_pct) * np.log((act_pct + eps) / (exp_pct + eps))
|
|
85
|
+
return {
|
|
86
|
+
"kind": "psi_test",
|
|
87
|
+
"psi": float(contributions.sum()),
|
|
88
|
+
"buckets": [
|
|
89
|
+
{"expected_pct": float(e), "actual_pct": float(a), "contribution": float(c)}
|
|
90
|
+
for e, a, c in zip(exp_pct, act_pct, contributions)
|
|
91
|
+
],
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def rating_summary(df: pl.DataFrame, grade_col: str, target_col: str) -> dict:
|
|
96
|
+
"""Per-grade population and observed default rate -- the standard check
|
|
97
|
+
that a rating scale (see modelling.fit_master_scale/assign_rating_grade)
|
|
98
|
+
is monotonic: default rate should rise, never fall, from the
|
|
99
|
+
lowest-risk grade to the highest."""
|
|
100
|
+
stats = df.group_by(grade_col).agg(n=pl.len(), default_rate=pl.col(target_col).mean())
|
|
101
|
+
stats = stats.sort(pl.col(grade_col).cast(pl.Int64, strict=False), nulls_last=True)
|
|
102
|
+
rows = stats.to_dicts()
|
|
103
|
+
rates = [r["default_rate"] for r in rows]
|
|
104
|
+
monotonic = all(a <= b for a, b in zip(rates, rates[1:]))
|
|
105
|
+
return {"kind": "rating_summary", "grades": rows, "monotonic": monotonic}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
register_block(
|
|
109
|
+
BlockSpec(
|
|
110
|
+
category="rating_summary",
|
|
111
|
+
block_type="output",
|
|
112
|
+
group="tests",
|
|
113
|
+
display_name="Rating scale summary",
|
|
114
|
+
inputs=[PortSpec("df")],
|
|
115
|
+
outputs=[PortSpec("metric", type="scalar_metric")],
|
|
116
|
+
fn=rating_summary,
|
|
117
|
+
metadata_transform=lambda *_a, **_k: {},
|
|
118
|
+
)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
register_block(
|
|
123
|
+
BlockSpec(
|
|
124
|
+
category="psi_test",
|
|
125
|
+
block_type="output",
|
|
126
|
+
group="tests",
|
|
127
|
+
display_name="PSI test",
|
|
128
|
+
inputs=[PortSpec("expected"), PortSpec("actual")],
|
|
129
|
+
outputs=[PortSpec("metric", type="scalar_metric")],
|
|
130
|
+
fn=psi_test,
|
|
131
|
+
metadata_transform=lambda *_a, **_k: {},
|
|
132
|
+
)
|
|
133
|
+
)
|
modelmaker/cache.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pickle
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class CacheEntry:
|
|
12
|
+
outputs: dict[str, Any]
|
|
13
|
+
timestamp: str
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CacheStore:
|
|
17
|
+
"""Keyed by lineage-based cache key (see Runner.compute_key) — never by
|
|
18
|
+
a hash of the actual data, so computing a key is cheap regardless of
|
|
19
|
+
DataFrame size. `cache_dir`, when given, is expected to be gitignored:
|
|
20
|
+
it holds run outputs, not the versioned project definition."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, cache_dir: Path | None = None):
|
|
23
|
+
self._mem: dict[str, CacheEntry] = {}
|
|
24
|
+
self._dir = cache_dir
|
|
25
|
+
if self._dir:
|
|
26
|
+
self._dir.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
|
|
28
|
+
def get(self, key: str) -> CacheEntry | None:
|
|
29
|
+
if key in self._mem:
|
|
30
|
+
return self._mem[key]
|
|
31
|
+
if self._dir:
|
|
32
|
+
p = self._dir / f"{key}.pkl"
|
|
33
|
+
if p.exists():
|
|
34
|
+
entry = pickle.loads(p.read_bytes())
|
|
35
|
+
self._mem[key] = entry
|
|
36
|
+
return entry
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
def set(self, key: str, outputs: dict[str, Any]) -> CacheEntry:
|
|
40
|
+
entry = CacheEntry(outputs=outputs, timestamp=datetime.now(timezone.utc).isoformat())
|
|
41
|
+
self._mem[key] = entry
|
|
42
|
+
if self._dir:
|
|
43
|
+
(self._dir / f"{key}.pkl").write_bytes(pickle.dumps(entry))
|
|
44
|
+
return entry
|