gitlabds 2.4.1__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlabds-2.4.1 → gitlabds-2.4.2}/PKG-INFO +14 -3
- {gitlabds-2.4.1 → gitlabds-2.4.2}/README.md +13 -2
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/record_benchmarking.py +34 -3
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds.egg-info/PKG-INFO +14 -3
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds.egg-info/SOURCES.txt +1 -0
- gitlabds-2.4.2/tests/test_record_benchmarking.py +182 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/LICENSE +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/__init__.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/baselines.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/config_generator.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/dummy.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/feature_descriptions.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/feature_reduction.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/insights.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/memory_optimization.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/missing.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/model_evaluator.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/monitoring_metrics.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/outliers.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/serving_features.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/snowflake_connectors.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/split_data.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds/trends.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds.egg-info/dependency_links.txt +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds.egg-info/requires.txt +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/gitlabds.egg-info/top_level.txt +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/pyproject.toml +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/setup.cfg +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/setup.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/tests/test_feature_reduction.py +0 -0
- {gitlabds-2.4.1 → gitlabds-2.4.2}/tests/test_snowflake_connectors.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlabds
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.2
|
|
4
4
|
Summary: GitLab Data Science Tools
|
|
5
5
|
Author-email: Kevin Dietz <kdietz@gitlab.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -914,7 +914,7 @@ shap_df = gitlabds.per_record_shap(
|
|
|
914
914
|
#### Description
|
|
915
915
|
"Change X -> +Y" counterfactual search, direction chosen empirically. For each actionable lever SHAP surfaced for a record, evaluates candidate values drawn from peer-group quantiles, re-scores the model at each, and keeps the single move that maximizes `gain` — the improvement in the model score, signed by `score_goal` so that bigger is always better. No monotonicity assumption is made.
|
|
916
916
|
|
|
917
|
-
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5):`
|
|
917
|
+
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5, prefer_nearest_within=None):`
|
|
918
918
|
|
|
919
919
|
#### Parameters:
|
|
920
920
|
- _**model**_ : trained model (as in `per_record_shap`). How probabilities are scored is chosen from the model object itself — `.predict_proba()` where available, otherwise a raw `xgboost.Booster` or `lightgbm.Booster` is scored natively. An unsupported type raises a `TypeError` naming it
|
|
@@ -926,10 +926,11 @@ shap_df = gitlabds.per_record_shap(
|
|
|
926
926
|
- _**direction**_ : optional per-feature constraint mapping feature -> `'increasing'` | `'decreasing'` | `'both'` | `'none'` (dict or Series). `'increasing'`/`'decreasing'` restrict candidates to moves in that direction (an achievability constraint); `'both'` allows either; `'none'` excludes the feature from lever evaluation entirely, for features with no meaningful good direction (categoricals, identifiers, context-only numerics). Features absent from the mapping, or `direction=None`, default to `'both'`. A blank/NaN raises — normalize it to `'none'` where the feature dictionary is read
|
|
927
927
|
- _**score_goal**_ : `"maximize"` if a higher predicted probability of class 1 is the good outcome (e.g. class 1 = uptier), `"minimize"` if it is the bad one (e.g. class 1 = churn). Sets the sign of `gain`, which drives best-move selection, the `min_gain` filter, `impact_bucket`, and the output ordering. This is the outcome's polarity only — `direction` above is a separate, feature-level concept and is unaffected. Default is `"maximize"`
|
|
928
928
|
- _**actionable_values**_ : values in `shap_df["actionable"]` treated as actionable levers
|
|
929
|
-
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds
|
|
929
|
+
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds. With `prefer_nearest_within` set, selection runs first, so this threshold applies to the gain of the chosen (nearer, lower-gain) candidate rather than the lever's best — a lever whose best gain clears `min_gain` can still be dropped on the nearer move it settled for
|
|
930
930
|
- _**impact_medium, impact_high**_ : `gain` thresholds bucketing each lever into Low/Medium/High `impact_bucket`. Only meaningful for `gain > 0` — under a negative `min_gain` the retained harmful moves stay `"Low"` (the level-crossing floor doesn't apply to them), so read `gain`'s sign rather than the bucket alone
|
|
931
931
|
- _**levels**_ : dict mapping label -> upper bound on P(class 1) for that level (e.g., default `{"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}` means `p <= 0.03` -> "poor fit", `0.03 < p <= 0.10` -> "good fit", `p > 0.10` -> "best fit"). Key order doesn't matter — levels are sorted by cutoff and labels follow that order — but the top level's cutoff must be `float("inf")`. Cutoffs are always in P(class 1) space regardless of `score_goal`, and `crosses_level` is a plain inequality, so name the labels for your own outcome: a churn model would pass something like `{"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")}`. Used to detect whether a move crosses a level boundary (any crossing move is floored at "Medium" impact)
|
|
932
932
|
- _**n_bins**_ : quantile bins for numeric `peer_group` columns
|
|
933
|
+
- _**prefer_nearest_within**_ : `None` (default) keeps the highest-gain candidate per (record, feature). A float in `(0, 1]` instead keeps the *nearest* candidate that still captures that fraction of the best gain: among candidates with `gain >= prefer_nearest_within * best_gain`, the one with the smallest `abs(intervention_value - current_value)` wins, ties broken on higher gain.
|
|
933
934
|
|
|
934
935
|
#### Returns
|
|
935
936
|
- Long DataFrame, one row per (record, lever): `current_value`, `intervention_value`, `intervention_quantile`, `p_before`, `p_after`, `delta_p`, `gain`, `impact_bucket`, `level_before`, `level_after`, `crosses_level`
|
|
@@ -960,6 +961,16 @@ counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
|
960
961
|
direction=dict(zip(feature_dict["field"], feature_dict["good_direction"])),
|
|
961
962
|
levels={"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")},
|
|
962
963
|
)
|
|
964
|
+
|
|
965
|
+
# Prefer a nearer, more sellable move: the closest candidate that still captures
|
|
966
|
+
# at least 70% of the best available gain.
|
|
967
|
+
counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
968
|
+
model=trained_model,
|
|
969
|
+
df=model_features,
|
|
970
|
+
shap_df=shap_df,
|
|
971
|
+
peer_group=peer_group_df,
|
|
972
|
+
prefer_nearest_within=0.7,
|
|
973
|
+
)
|
|
963
974
|
```
|
|
964
975
|
|
|
965
976
|
</details>
|
|
@@ -878,7 +878,7 @@ shap_df = gitlabds.per_record_shap(
|
|
|
878
878
|
#### Description
|
|
879
879
|
"Change X -> +Y" counterfactual search, direction chosen empirically. For each actionable lever SHAP surfaced for a record, evaluates candidate values drawn from peer-group quantiles, re-scores the model at each, and keeps the single move that maximizes `gain` — the improvement in the model score, signed by `score_goal` so that bigger is always better. No monotonicity assumption is made.
|
|
880
880
|
|
|
881
|
-
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5):`
|
|
881
|
+
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5, prefer_nearest_within=None):`
|
|
882
882
|
|
|
883
883
|
#### Parameters:
|
|
884
884
|
- _**model**_ : trained model (as in `per_record_shap`). How probabilities are scored is chosen from the model object itself — `.predict_proba()` where available, otherwise a raw `xgboost.Booster` or `lightgbm.Booster` is scored natively. An unsupported type raises a `TypeError` naming it
|
|
@@ -890,10 +890,11 @@ shap_df = gitlabds.per_record_shap(
|
|
|
890
890
|
- _**direction**_ : optional per-feature constraint mapping feature -> `'increasing'` | `'decreasing'` | `'both'` | `'none'` (dict or Series). `'increasing'`/`'decreasing'` restrict candidates to moves in that direction (an achievability constraint); `'both'` allows either; `'none'` excludes the feature from lever evaluation entirely, for features with no meaningful good direction (categoricals, identifiers, context-only numerics). Features absent from the mapping, or `direction=None`, default to `'both'`. A blank/NaN raises — normalize it to `'none'` where the feature dictionary is read
|
|
891
891
|
- _**score_goal**_ : `"maximize"` if a higher predicted probability of class 1 is the good outcome (e.g. class 1 = uptier), `"minimize"` if it is the bad one (e.g. class 1 = churn). Sets the sign of `gain`, which drives best-move selection, the `min_gain` filter, `impact_bucket`, and the output ordering. This is the outcome's polarity only — `direction` above is a separate, feature-level concept and is unaffected. Default is `"maximize"`
|
|
892
892
|
- _**actionable_values**_ : values in `shap_df["actionable"]` treated as actionable levers
|
|
893
|
-
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds
|
|
893
|
+
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds. With `prefer_nearest_within` set, selection runs first, so this threshold applies to the gain of the chosen (nearer, lower-gain) candidate rather than the lever's best — a lever whose best gain clears `min_gain` can still be dropped on the nearer move it settled for
|
|
894
894
|
- _**impact_medium, impact_high**_ : `gain` thresholds bucketing each lever into Low/Medium/High `impact_bucket`. Only meaningful for `gain > 0` — under a negative `min_gain` the retained harmful moves stay `"Low"` (the level-crossing floor doesn't apply to them), so read `gain`'s sign rather than the bucket alone
|
|
895
895
|
- _**levels**_ : dict mapping label -> upper bound on P(class 1) for that level (e.g., default `{"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}` means `p <= 0.03` -> "poor fit", `0.03 < p <= 0.10` -> "good fit", `p > 0.10` -> "best fit"). Key order doesn't matter — levels are sorted by cutoff and labels follow that order — but the top level's cutoff must be `float("inf")`. Cutoffs are always in P(class 1) space regardless of `score_goal`, and `crosses_level` is a plain inequality, so name the labels for your own outcome: a churn model would pass something like `{"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")}`. Used to detect whether a move crosses a level boundary (any crossing move is floored at "Medium" impact)
|
|
896
896
|
- _**n_bins**_ : quantile bins for numeric `peer_group` columns
|
|
897
|
+
- _**prefer_nearest_within**_ : `None` (default) keeps the highest-gain candidate per (record, feature). A float in `(0, 1]` instead keeps the *nearest* candidate that still captures that fraction of the best gain: among candidates with `gain >= prefer_nearest_within * best_gain`, the one with the smallest `abs(intervention_value - current_value)` wins, ties broken on higher gain.
|
|
897
898
|
|
|
898
899
|
#### Returns
|
|
899
900
|
- Long DataFrame, one row per (record, lever): `current_value`, `intervention_value`, `intervention_quantile`, `p_before`, `p_after`, `delta_p`, `gain`, `impact_bucket`, `level_before`, `level_after`, `crosses_level`
|
|
@@ -924,6 +925,16 @@ counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
|
924
925
|
direction=dict(zip(feature_dict["field"], feature_dict["good_direction"])),
|
|
925
926
|
levels={"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")},
|
|
926
927
|
)
|
|
928
|
+
|
|
929
|
+
# Prefer a nearer, more sellable move: the closest candidate that still captures
|
|
930
|
+
# at least 70% of the best available gain.
|
|
931
|
+
counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
932
|
+
model=trained_model,
|
|
933
|
+
df=model_features,
|
|
934
|
+
shap_df=shap_df,
|
|
935
|
+
peer_group=peer_group_df,
|
|
936
|
+
prefer_nearest_within=0.7,
|
|
937
|
+
)
|
|
927
938
|
```
|
|
928
939
|
|
|
929
940
|
</details>
|
|
@@ -297,6 +297,7 @@ def per_record_counterfactuals(
|
|
|
297
297
|
impact_high=0.03,
|
|
298
298
|
levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")},
|
|
299
299
|
n_bins=5,
|
|
300
|
+
prefer_nearest_within=None,
|
|
300
301
|
):
|
|
301
302
|
"""
|
|
302
303
|
"change X -> +Y", direction chosen empirically.
|
|
@@ -335,6 +336,10 @@ def per_record_counterfactuals(
|
|
|
335
336
|
a negative delta_p but a positive gain, so this stays positive for both polarities.
|
|
336
337
|
Negative values (or -inf) are allowed, and keep moves that make the outcome worse --
|
|
337
338
|
useful for inspecting the full candidate distribution while tuning thresholds.
|
|
339
|
+
With prefer_nearest_within set, selection runs first, so this threshold applies to
|
|
340
|
+
the gain of the chosen (nearer, lower-gain) candidate rather than the lever's best:
|
|
341
|
+
a lever whose best gain clears min_gain can still be dropped on the nearer move it
|
|
342
|
+
settled for.
|
|
338
343
|
impact_medium, impact_high : gain thresholds bucketing each lever into
|
|
339
344
|
Low/Medium/High impact_bucket. gain shrinks mechanically near the tails of
|
|
340
345
|
the probability distribution, so a fixed pp threshold alone under-weights
|
|
@@ -360,6 +365,11 @@ def per_record_counterfactuals(
|
|
|
360
365
|
"high risk": float("inf")}. Used only to detect level crossings for impact_bucket,
|
|
361
366
|
not to reproduce your production leveling logic.
|
|
362
367
|
n_bins : quantile bins for numeric peer_group columns.
|
|
368
|
+
prefer_nearest_within : None (default) keeps the highest-gain candidate per (record,
|
|
369
|
+
feature) -- the historical behavior. A float in (0, 1] instead keeps the *nearest*
|
|
370
|
+
candidate that still captures that fraction of the best gain: among candidates with
|
|
371
|
+
gain >= prefer_nearest_within * best_gain, the one with the smallest
|
|
372
|
+
abs(intervention_value - current_value) wins, ties broken on higher gain.
|
|
363
373
|
|
|
364
374
|
Returns a long DataFrame: one row per (record, lever) with the best move found
|
|
365
375
|
(current_value, intervention_value, intervention_quantile, p_before/p_after/delta_p,
|
|
@@ -368,6 +378,13 @@ def per_record_counterfactuals(
|
|
|
368
378
|
"""
|
|
369
379
|
if impact_medium >= impact_high:
|
|
370
380
|
raise ValueError(f"impact_medium ({impact_medium}) must be less than impact_high ({impact_high})")
|
|
381
|
+
if prefer_nearest_within is not None:
|
|
382
|
+
prefer_nearest_within = float(prefer_nearest_within)
|
|
383
|
+
if not 0 < prefer_nearest_within <= 1:
|
|
384
|
+
raise ValueError(
|
|
385
|
+
"prefer_nearest_within must be greater than 0 and at most 1 (the fraction of "
|
|
386
|
+
f"the best gain a nearer candidate must still capture), got {prefer_nearest_within}"
|
|
387
|
+
)
|
|
371
388
|
_score_goal = str(score_goal).strip().lower()
|
|
372
389
|
if _score_goal not in ("maximize", "minimize"):
|
|
373
390
|
raise ValueError(f"score_goal must be 'maximize' or 'minimize', got {score_goal!r}")
|
|
@@ -442,9 +459,23 @@ def per_record_counterfactuals(
|
|
|
442
459
|
return pd.DataFrame()
|
|
443
460
|
out = pd.concat(parts, ignore_index=True)
|
|
444
461
|
out["gain"] = sign * out["delta_p"]
|
|
445
|
-
# Keep
|
|
446
|
-
|
|
447
|
-
|
|
462
|
+
# Keep one candidate per (record, feature): the best-scoring one, or -- with
|
|
463
|
+
# prefer_nearest_within -- the nearest one that still captures that share of the best
|
|
464
|
+
# gain. Done before the min_gain filter and the impact/level logic below, so all of
|
|
465
|
+
# those describe the move actually chosen.
|
|
466
|
+
if prefer_nearest_within is None:
|
|
467
|
+
out = (out.sort_values("gain", ascending=False)
|
|
468
|
+
.drop_duplicates([unique_id, "feature"], keep="first"))
|
|
469
|
+
else:
|
|
470
|
+
best = out.groupby([unique_id, "feature"], observed=True)["gain"].transform("max")
|
|
471
|
+
# Scaling only makes sense for a positive best gain: at best_gain <= 0, multiplying
|
|
472
|
+
# by a fraction raises the bar above the best candidate and would empty the group.
|
|
473
|
+
out = out[out["gain"] >= np.where(best > 0, prefer_nearest_within * best, best)]
|
|
474
|
+
distance = (out["intervention_value"] - out["current_value"]).abs()
|
|
475
|
+
out = (out.assign(_distance=distance)
|
|
476
|
+
.sort_values(["_distance", "gain"], ascending=[True, False])
|
|
477
|
+
.drop_duplicates([unique_id, "feature"], keep="first")
|
|
478
|
+
.drop(columns="_distance"))
|
|
448
479
|
out = out[out["gain"] >= min_gain]
|
|
449
480
|
out["impact_bucket"] = pd.cut(out["gain"],
|
|
450
481
|
bins=[-np.inf, impact_medium, impact_high, np.inf],
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlabds
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.2
|
|
4
4
|
Summary: GitLab Data Science Tools
|
|
5
5
|
Author-email: Kevin Dietz <kdietz@gitlab.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -914,7 +914,7 @@ shap_df = gitlabds.per_record_shap(
|
|
|
914
914
|
#### Description
|
|
915
915
|
"Change X -> +Y" counterfactual search, direction chosen empirically. For each actionable lever SHAP surfaced for a record, evaluates candidate values drawn from peer-group quantiles, re-scores the model at each, and keeps the single move that maximizes `gain` — the improvement in the model score, signed by `score_goal` so that bigger is always better. No monotonicity assumption is made.
|
|
916
916
|
|
|
917
|
-
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5):`
|
|
917
|
+
`gitlabds.per_record_counterfactuals(model, df, shap_df, unique_id="record_id", peer_group=None, candidate_quantiles=[0.1, 0.25, 0.5, 0.75, 0.9], direction=None, score_goal="maximize", actionable_values=["Yes", "Partial"], min_gain=0.005, impact_medium=0.01, impact_high=0.03, levels={"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}, n_bins=5, prefer_nearest_within=None):`
|
|
918
918
|
|
|
919
919
|
#### Parameters:
|
|
920
920
|
- _**model**_ : trained model (as in `per_record_shap`). How probabilities are scored is chosen from the model object itself — `.predict_proba()` where available, otherwise a raw `xgboost.Booster` or `lightgbm.Booster` is scored natively. An unsupported type raises a `TypeError` naming it
|
|
@@ -926,10 +926,11 @@ shap_df = gitlabds.per_record_shap(
|
|
|
926
926
|
- _**direction**_ : optional per-feature constraint mapping feature -> `'increasing'` | `'decreasing'` | `'both'` | `'none'` (dict or Series). `'increasing'`/`'decreasing'` restrict candidates to moves in that direction (an achievability constraint); `'both'` allows either; `'none'` excludes the feature from lever evaluation entirely, for features with no meaningful good direction (categoricals, identifiers, context-only numerics). Features absent from the mapping, or `direction=None`, default to `'both'`. A blank/NaN raises — normalize it to `'none'` where the feature dictionary is read
|
|
927
927
|
- _**score_goal**_ : `"maximize"` if a higher predicted probability of class 1 is the good outcome (e.g. class 1 = uptier), `"minimize"` if it is the bad one (e.g. class 1 = churn). Sets the sign of `gain`, which drives best-move selection, the `min_gain` filter, `impact_bucket`, and the output ordering. This is the outcome's polarity only — `direction` above is a separate, feature-level concept and is unaffected. Default is `"maximize"`
|
|
928
928
|
- _**actionable_values**_ : values in `shap_df["actionable"]` treated as actionable levers
|
|
929
|
-
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds
|
|
929
|
+
- _**min_gain**_ : drop levers whose best achievable gain is below this. Note this is a threshold on `gain`, not on raw `delta_p`: under `score_goal="minimize"` a good move has a negative `delta_p` but a positive `gain`, so this stays positive for both polarities. Negative values (or `-inf`) are allowed, and keep moves that make the outcome worse — useful for inspecting the full candidate distribution while tuning thresholds. With `prefer_nearest_within` set, selection runs first, so this threshold applies to the gain of the chosen (nearer, lower-gain) candidate rather than the lever's best — a lever whose best gain clears `min_gain` can still be dropped on the nearer move it settled for
|
|
930
930
|
- _**impact_medium, impact_high**_ : `gain` thresholds bucketing each lever into Low/Medium/High `impact_bucket`. Only meaningful for `gain > 0` — under a negative `min_gain` the retained harmful moves stay `"Low"` (the level-crossing floor doesn't apply to them), so read `gain`'s sign rather than the bucket alone
|
|
931
931
|
- _**levels**_ : dict mapping label -> upper bound on P(class 1) for that level (e.g., default `{"poor fit": 0.03, "good fit": 0.10, "best fit": float("inf")}` means `p <= 0.03` -> "poor fit", `0.03 < p <= 0.10` -> "good fit", `p > 0.10` -> "best fit"). Key order doesn't matter — levels are sorted by cutoff and labels follow that order — but the top level's cutoff must be `float("inf")`. Cutoffs are always in P(class 1) space regardless of `score_goal`, and `crosses_level` is a plain inequality, so name the labels for your own outcome: a churn model would pass something like `{"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")}`. Used to detect whether a move crosses a level boundary (any crossing move is floored at "Medium" impact)
|
|
932
932
|
- _**n_bins**_ : quantile bins for numeric `peer_group` columns
|
|
933
|
+
- _**prefer_nearest_within**_ : `None` (default) keeps the highest-gain candidate per (record, feature). A float in `(0, 1]` instead keeps the *nearest* candidate that still captures that fraction of the best gain: among candidates with `gain >= prefer_nearest_within * best_gain`, the one with the smallest `abs(intervention_value - current_value)` wins, ties broken on higher gain.
|
|
933
934
|
|
|
934
935
|
#### Returns
|
|
935
936
|
- Long DataFrame, one row per (record, lever): `current_value`, `intervention_value`, `intervention_quantile`, `p_before`, `p_after`, `delta_p`, `gain`, `impact_bucket`, `level_before`, `level_after`, `crosses_level`
|
|
@@ -960,6 +961,16 @@ counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
|
960
961
|
direction=dict(zip(feature_dict["field"], feature_dict["good_direction"])),
|
|
961
962
|
levels={"low risk": 0.03, "medium risk": 0.10, "high risk": float("inf")},
|
|
962
963
|
)
|
|
964
|
+
|
|
965
|
+
# Prefer a nearer, more sellable move: the closest candidate that still captures
|
|
966
|
+
# at least 70% of the best available gain.
|
|
967
|
+
counterfactuals_df = gitlabds.per_record_counterfactuals(
|
|
968
|
+
model=trained_model,
|
|
969
|
+
df=model_features,
|
|
970
|
+
shap_df=shap_df,
|
|
971
|
+
peer_group=peer_group_df,
|
|
972
|
+
prefer_nearest_within=0.7,
|
|
973
|
+
)
|
|
963
974
|
```
|
|
964
975
|
|
|
965
976
|
</details>
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unit tests for gitlabds.record_benchmarking. No model is trained and `shap` is never
|
|
3
|
+
called: `per_record_counterfactuals` only needs an object exposing .predict_proba() and a
|
|
4
|
+
shap_df-shaped frame naming the (record, feature) levers to test, so both are faked here
|
|
5
|
+
with a closed-form probability that makes each candidate's gain predictable.
|
|
6
|
+
|
|
7
|
+
Run with: pytest tests/test_record_benchmarking.py
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from gitlabds.record_benchmarking import per_record_counterfactuals
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# ---------------------------------------------------------------------------
|
|
18
|
+
# Fixtures and fakes
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class RampModel:
|
|
23
|
+
"""
|
|
24
|
+
P(class 1) = base + slope * f, i.e. monotonic in the single feature "f" with
|
|
25
|
+
diminishing *relative* returns per unit moved. Lets each candidate quantile's gain be
|
|
26
|
+
computed by hand, so a test can assert which move the selection logic picked.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, base=0.0, slope=0.05):
|
|
30
|
+
self.base = base
|
|
31
|
+
self.slope = slope
|
|
32
|
+
|
|
33
|
+
def predict_proba(self, X):
|
|
34
|
+
p1 = np.clip(self.base + self.slope * np.asarray(X["f"], dtype=float), 0.0, 1.0)
|
|
35
|
+
return np.column_stack([1 - p1, p1])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.fixture
|
|
39
|
+
def df():
|
|
40
|
+
"""f = 0..9, so the global peer quantiles are p25=2.25, p50=4.5, p90=8.1."""
|
|
41
|
+
return pd.DataFrame(
|
|
42
|
+
{"f": np.arange(10, dtype=float)},
|
|
43
|
+
index=pd.Index([f"r{i}" for i in range(10)], name="record_id"),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def make_shap_df(records, feature="f"):
|
|
48
|
+
"""Minimal stand-in for per_record_shap output: enough to scope the levers."""
|
|
49
|
+
return pd.DataFrame({
|
|
50
|
+
"record_id": list(records),
|
|
51
|
+
"feature": feature,
|
|
52
|
+
"actionable": "Yes",
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def run(df, records, **kwargs):
|
|
57
|
+
return per_record_counterfactuals(
|
|
58
|
+
model=RampModel(),
|
|
59
|
+
df=df,
|
|
60
|
+
shap_df=make_shap_df(records),
|
|
61
|
+
candidate_quantiles=[0.25, 0.5, 0.9],
|
|
62
|
+
**kwargs,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
# Default behavior
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class TestDefaultSelection:
|
|
72
|
+
def test_keeps_the_highest_gain_candidate(self, df):
|
|
73
|
+
# r0 (f=0) can move to 2.25, 4.5 or 8.1; the ramp makes the farthest the best.
|
|
74
|
+
out = run(df, ["r0"])
|
|
75
|
+
assert len(out) == 1
|
|
76
|
+
row = out.iloc[0]
|
|
77
|
+
assert row["intervention_quantile"] == 0.9
|
|
78
|
+
assert row["intervention_value"] == pytest.approx(8.1)
|
|
79
|
+
assert row["gain"] == pytest.approx(0.405)
|
|
80
|
+
|
|
81
|
+
def test_prefer_nearest_within_none_matches_the_default(self, df):
|
|
82
|
+
records = ["r0", "r1", "r2"]
|
|
83
|
+
pd.testing.assert_frame_equal(
|
|
84
|
+
run(df, records),
|
|
85
|
+
run(df, records, prefer_nearest_within=None),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# prefer_nearest_within
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class TestPreferNearestWithin:
|
|
95
|
+
def test_nearer_candidate_within_the_threshold_wins(self, df):
|
|
96
|
+
# Gains for r0 are 0.1125 (2.25), 0.225 (4.5) and 0.405 (8.1). At 0.5, only the
|
|
97
|
+
# last two clear 0.2025, so the nearer of those -- the median -- is chosen.
|
|
98
|
+
out = run(df, ["r0"], prefer_nearest_within=0.5)
|
|
99
|
+
assert len(out) == 1
|
|
100
|
+
row = out.iloc[0]
|
|
101
|
+
assert row["intervention_quantile"] == 0.5
|
|
102
|
+
assert row["intervention_value"] == pytest.approx(4.5)
|
|
103
|
+
assert row["gain"] == pytest.approx(0.225)
|
|
104
|
+
|
|
105
|
+
def test_threshold_too_high_leaves_the_best_gain_move(self, df):
|
|
106
|
+
# 0.9 * 0.405 = 0.3645, which only the best candidate clears.
|
|
107
|
+
out = run(df, ["r0"], prefer_nearest_within=0.9)
|
|
108
|
+
assert out.iloc[0]["intervention_value"] == pytest.approx(8.1)
|
|
109
|
+
|
|
110
|
+
def test_lower_threshold_reaches_a_nearer_candidate(self, df):
|
|
111
|
+
# 0.25 * 0.405 = 0.101, low enough to admit p25 as well.
|
|
112
|
+
out = run(df, ["r0"], prefer_nearest_within=0.25)
|
|
113
|
+
assert out.iloc[0]["intervention_value"] == pytest.approx(2.25)
|
|
114
|
+
|
|
115
|
+
def test_one_row_per_record_feature_like_the_default(self, df):
|
|
116
|
+
records = ["r0", "r1", "r2", "r3"]
|
|
117
|
+
default = run(df, records)
|
|
118
|
+
nearest = run(df, records, prefer_nearest_within=0.5)
|
|
119
|
+
assert list(nearest.columns) == list(default.columns)
|
|
120
|
+
assert set(nearest["record_id"]) == set(default["record_id"])
|
|
121
|
+
assert not nearest.duplicated(["record_id", "feature"]).any()
|
|
122
|
+
|
|
123
|
+
def test_impact_bucket_describes_the_chosen_move(self, df):
|
|
124
|
+
# The bucket is computed after selection, so it reflects the nearer move's gain --
|
|
125
|
+
# here a full bucket below what the default pick would have earned. A single level
|
|
126
|
+
# keeps the level-crossing floor from promoting it, so the bucket is the gain's.
|
|
127
|
+
kwargs = dict(impact_medium=0.25, impact_high=0.35, levels={"same level": float("inf")})
|
|
128
|
+
out = run(df, ["r0"], prefer_nearest_within=0.5, **kwargs)
|
|
129
|
+
assert out.iloc[0]["gain"] == pytest.approx(0.225)
|
|
130
|
+
assert not out.iloc[0]["crosses_level"]
|
|
131
|
+
assert out.iloc[0]["impact_bucket"] == "Low"
|
|
132
|
+
assert run(df, ["r0"], **kwargs).iloc[0]["impact_bucket"] == "High"
|
|
133
|
+
|
|
134
|
+
def test_single_candidate_record_is_unaffected(self, df):
|
|
135
|
+
# r5 (f=5) constrained upward has only p90 (8.1) available.
|
|
136
|
+
kwargs = dict(direction={"f": "increasing"})
|
|
137
|
+
default = run(df, ["r5"], **kwargs)
|
|
138
|
+
nearest = run(df, ["r5"], prefer_nearest_within=0.5, **kwargs)
|
|
139
|
+
assert len(default) == 1
|
|
140
|
+
assert default.iloc[0]["intervention_value"] == pytest.approx(8.1)
|
|
141
|
+
pd.testing.assert_frame_equal(default, nearest)
|
|
142
|
+
|
|
143
|
+
def test_minimize_goal_picks_the_nearer_candidate(self, df):
|
|
144
|
+
# Class 1 is the bad outcome here: the ramp runs downward, so every move lowers
|
|
145
|
+
# p and gain = -delta_p. Same gains as the maximize case, same expected pick.
|
|
146
|
+
model = RampModel(base=0.5, slope=-0.05)
|
|
147
|
+
kwargs = dict(
|
|
148
|
+
model=model, df=df, shap_df=make_shap_df(["r0"]),
|
|
149
|
+
candidate_quantiles=[0.25, 0.5, 0.9], score_goal="minimize",
|
|
150
|
+
)
|
|
151
|
+
default = per_record_counterfactuals(**kwargs)
|
|
152
|
+
nearest = per_record_counterfactuals(**kwargs, prefer_nearest_within=0.5)
|
|
153
|
+
assert default.iloc[0]["intervention_value"] == pytest.approx(8.1)
|
|
154
|
+
assert nearest.iloc[0]["intervention_value"] == pytest.approx(4.5)
|
|
155
|
+
assert nearest.iloc[0]["delta_p"] == pytest.approx(-0.225)
|
|
156
|
+
assert nearest.iloc[0]["gain"] == pytest.approx(0.225)
|
|
157
|
+
|
|
158
|
+
def test_harmful_moves_keep_their_best_candidate(self, df):
|
|
159
|
+
# Under a negative min_gain every candidate for r9 (f=9) makes the outcome worse,
|
|
160
|
+
# so best_gain is negative; scaling it would drop the lever entirely.
|
|
161
|
+
kwargs = dict(min_gain=-np.inf, direction={"f": "decreasing"})
|
|
162
|
+
default = run(df, ["r9"], **kwargs)
|
|
163
|
+
nearest = run(df, ["r9"], prefer_nearest_within=0.5, **kwargs)
|
|
164
|
+
assert len(nearest) == 1
|
|
165
|
+
assert nearest.iloc[0]["gain"] < 0
|
|
166
|
+
pd.testing.assert_frame_equal(default, nearest)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class TestPreferNearestWithinValidation:
|
|
170
|
+
@pytest.mark.parametrize("value", [0, -0.1, 1.5, 2])
|
|
171
|
+
def test_out_of_range_raises(self, df, value):
|
|
172
|
+
with pytest.raises(ValueError, match="prefer_nearest_within"):
|
|
173
|
+
run(df, ["r0"], prefer_nearest_within=value)
|
|
174
|
+
|
|
175
|
+
def test_non_numeric_raises(self, df):
|
|
176
|
+
with pytest.raises(ValueError):
|
|
177
|
+
run(df, ["r0"], prefer_nearest_within="most")
|
|
178
|
+
|
|
179
|
+
def test_one_is_allowed(self, df):
|
|
180
|
+
# 1.0 means "only candidates tied with the best gain", i.e. the default pick.
|
|
181
|
+
out = run(df, ["r0"], prefer_nearest_within=1.0)
|
|
182
|
+
assert out.iloc[0]["intervention_value"] == pytest.approx(8.1)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|