py-flexplot 0.8.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_flexplot-0.8.2.dist-info/METADATA +272 -0
- py_flexplot-0.8.2.dist-info/RECORD +16 -0
- py_flexplot-0.8.2.dist-info/WHEEL +5 -0
- py_flexplot-0.8.2.dist-info/licenses/LICENSE +21 -0
- py_flexplot-0.8.2.dist-info/top_level.txt +1 -0
- pyflexplot/__init__.py +60 -0
- pyflexplot/bluepill.py +461 -0
- pyflexplot/core.py +3410 -0
- pyflexplot/descriptives.py +287 -0
- pyflexplot/ebbr.py +148 -0
- pyflexplot/flex_nn.py +583 -0
- pyflexplot/ml.py +175 -0
- pyflexplot/quality.py +372 -0
- pyflexplot/sem.py +195 -0
- pyflexplot/stats.py +803 -0
- pyflexplot/uncertainty.py +185 -0
pyflexplot/core.py
ADDED
|
@@ -0,0 +1,3410 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
import warnings
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import numpy as np
|
|
7
|
+
from typing import Any, List, Optional, Union, cast
|
|
8
|
+
from plotnine import (
|
|
9
|
+
ggplot,
|
|
10
|
+
aes,
|
|
11
|
+
geom_histogram,
|
|
12
|
+
geom_point,
|
|
13
|
+
geom_smooth,
|
|
14
|
+
geom_jitter,
|
|
15
|
+
geom_line,
|
|
16
|
+
geom_ribbon,
|
|
17
|
+
geom_hline,
|
|
18
|
+
geom_boxplot,
|
|
19
|
+
geom_violin,
|
|
20
|
+
geom_bar,
|
|
21
|
+
geom_density,
|
|
22
|
+
stat_summary,
|
|
23
|
+
stat_qq,
|
|
24
|
+
stat_qq_line,
|
|
25
|
+
facet_wrap,
|
|
26
|
+
facet_grid,
|
|
27
|
+
scale_color_identity,
|
|
28
|
+
scale_color_manual,
|
|
29
|
+
labs,
|
|
30
|
+
theme_bw,
|
|
31
|
+
theme,
|
|
32
|
+
element_blank,
|
|
33
|
+
)
|
|
34
|
+
import statsmodels.api as sm
|
|
35
|
+
import statsmodels.formula.api as smf
|
|
36
|
+
from statsmodels.regression.linear_model import OLS
|
|
37
|
+
from statsmodels.nonparametric.smoothers_lowess import lowess
|
|
38
|
+
|
|
39
|
+
from .uncertainty import (
|
|
40
|
+
validate_uncertainty_params,
|
|
41
|
+
compute_bootstrap_ci,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_flexplot_formula(formula: str):
|
|
46
|
+
"""
|
|
47
|
+
Parses a flexplot formula of the form:
|
|
48
|
+
outcome ~ predictor1 + predictor2 | given1 + given2
|
|
49
|
+
|
|
50
|
+
Validates the formula syntax, strips whitespace from tokens, rejects empty
|
|
51
|
+
outcome/predictor, handles the intercept-only form ``y ~ 1`` explicitly,
|
|
52
|
+
and allows at most one ``|``.
|
|
53
|
+
"""
|
|
54
|
+
if not isinstance(formula, str):
|
|
55
|
+
raise TypeError(f"formula must be a string, got {type(formula).__name__}")
|
|
56
|
+
|
|
57
|
+
if formula.count("~") != 1:
|
|
58
|
+
raise ValueError(
|
|
59
|
+
f"Formula must contain exactly one '~': {formula!r}"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
if formula.count("|") > 1:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Formula may contain at most one '|': {formula!r}"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if "|" in formula:
|
|
68
|
+
main_part, given_part = formula.split("|", 1)
|
|
69
|
+
else:
|
|
70
|
+
main_part = formula
|
|
71
|
+
given_part = None
|
|
72
|
+
|
|
73
|
+
main_part = main_part.strip()
|
|
74
|
+
given_part = given_part.strip() if given_part is not None else None
|
|
75
|
+
|
|
76
|
+
y_name, sep, x_formula = main_part.partition("~")
|
|
77
|
+
y_name = y_name.strip()
|
|
78
|
+
x_formula = x_formula.strip()
|
|
79
|
+
|
|
80
|
+
if not y_name:
|
|
81
|
+
raise ValueError(f"Formula must have a non-empty outcome: {formula!r}")
|
|
82
|
+
if not x_formula:
|
|
83
|
+
raise ValueError(
|
|
84
|
+
f"Formula must have predictors after '~' (use 'y ~ 1' for intercept-only): {formula!r}"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Intercept-only formula: y ~ 1
|
|
88
|
+
if x_formula == "1":
|
|
89
|
+
return {
|
|
90
|
+
"y": y_name,
|
|
91
|
+
"x": None,
|
|
92
|
+
"color": None,
|
|
93
|
+
"given": [g.strip() for g in given_part.split("+")] if given_part else [],
|
|
94
|
+
"all_x": [],
|
|
95
|
+
"intercept_only": True,
|
|
96
|
+
"has_interaction": False,
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
# Detect interaction operators (``*`` or ``:``) anywhere in the
|
|
100
|
+
# right-hand side. The current fit is still additive; we set a flag
|
|
101
|
+
# so ``flexplot()`` can warn the user. ``*`` is expanded to its R-style
|
|
102
|
+
# constituent terms (``a*b`` → ``a + b + a:b``) so column lookup works.
|
|
103
|
+
has_interaction = bool(_INTERACTION_OP.search(x_formula))
|
|
104
|
+
if has_interaction:
|
|
105
|
+
expanded_x_formula = _expand_r_formula(x_formula)
|
|
106
|
+
else:
|
|
107
|
+
expanded_x_formula = x_formula
|
|
108
|
+
|
|
109
|
+
x_parts = [p.strip() for p in expanded_x_formula.split("+")]
|
|
110
|
+
x_parts = [p for p in x_parts if p]
|
|
111
|
+
|
|
112
|
+
if not x_parts:
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f"Formula must have at least one predictor after '~': {formula!r}"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# ``x_name`` is the first atom of the first term (so ``x:z`` → ``x``);
|
|
118
|
+
# ``color_name`` is the first atom of the second term if present.
|
|
119
|
+
x_name = _first_atom(x_parts[0])
|
|
120
|
+
color_name = _first_atom(x_parts[1]) if len(x_parts) > 1 else None
|
|
121
|
+
|
|
122
|
+
given_names = [g.strip() for g in given_part.split("+")] if given_part else []
|
|
123
|
+
given_names = [g for g in given_names if g]
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
"y": y_name,
|
|
127
|
+
"x": x_name,
|
|
128
|
+
"color": color_name,
|
|
129
|
+
"given": given_names,
|
|
130
|
+
"all_x": x_parts,
|
|
131
|
+
"intercept_only": False,
|
|
132
|
+
"has_interaction": has_interaction,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _validate_data_for_plot(
|
|
137
|
+
formula: str,
|
|
138
|
+
data: pd.DataFrame,
|
|
139
|
+
variables: dict,
|
|
140
|
+
require_numeric_x: bool = False,
|
|
141
|
+
intercept_only: bool = False,
|
|
142
|
+
):
|
|
143
|
+
"""Shared validation for flexplot and added_plot."""
|
|
144
|
+
if not isinstance(data, pd.DataFrame):
|
|
145
|
+
raise TypeError(
|
|
146
|
+
f"data must be a pandas DataFrame, got {type(data).__name__}"
|
|
147
|
+
)
|
|
148
|
+
if data.empty:
|
|
149
|
+
raise ValueError(
|
|
150
|
+
f"data must be a non-empty DataFrame for formula {formula!r}"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
y = variables["y"]
|
|
154
|
+
x = variables["x"]
|
|
155
|
+
given = variables.get("given", [])
|
|
156
|
+
color = variables.get("color")
|
|
157
|
+
|
|
158
|
+
required = {y}
|
|
159
|
+
if x is not None:
|
|
160
|
+
required.add(x)
|
|
161
|
+
if color is not None:
|
|
162
|
+
required.add(color)
|
|
163
|
+
for g in given:
|
|
164
|
+
required.add(g)
|
|
165
|
+
|
|
166
|
+
missing = sorted(required - set(data.columns))
|
|
167
|
+
if missing:
|
|
168
|
+
raise ValueError(
|
|
169
|
+
f"Formula {formula!r} references missing columns in data: {missing}"
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# Validate outcome column y is numeric (or numeric-convertible).
|
|
173
|
+
# For intercept-only formulas, R-flexplot supports univariate plots of
|
|
174
|
+
# categorical outcomes (bar charts), so we relax the numeric requirement.
|
|
175
|
+
if (
|
|
176
|
+
y is not None
|
|
177
|
+
and not intercept_only
|
|
178
|
+
and not pd.api.types.is_numeric_dtype(data[y])
|
|
179
|
+
):
|
|
180
|
+
try:
|
|
181
|
+
pd.to_numeric(data[y].dropna())
|
|
182
|
+
except (ValueError, TypeError):
|
|
183
|
+
raise ValueError(
|
|
184
|
+
f"Column {y!r} must be numeric for formula {formula!r}, "
|
|
185
|
+
f"got dtype {data[y].dtype}"
|
|
186
|
+
) from None
|
|
187
|
+
|
|
188
|
+
if require_numeric_x and x is not None and not pd.api.types.is_numeric_dtype(data[x]):
|
|
189
|
+
raise ValueError(
|
|
190
|
+
f"Column {x!r} must be numeric for formula {formula!r}, "
|
|
191
|
+
f"got dtype {data[x].dtype}"
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# Threshold below which R-flexplot converts numeric predictors to ordered
|
|
196
|
+
# categorical factors. Matches R's ``convert_if_less_than_five`` (numeric
|
|
197
|
+
# with <5 unique values -> ordered factor).
|
|
198
|
+
_LOW_CARDINALITY_THRESHOLD = 5
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _is_low_cardinality_numeric(series: pd.Series) -> bool:
|
|
202
|
+
"""Return True if ``series`` is numeric with fewer than 5 unique non-null values.
|
|
203
|
+
|
|
204
|
+
Mirrors R-flexplot's ``convert_if_less_than_five``: numeric axis / color /
|
|
205
|
+
given variables with 2-4 unique values should be treated as ordered
|
|
206
|
+
categorical factors, both for plotting and for the smoother fit. Series
|
|
207
|
+
that are already non-numeric, or numeric with 5+ unique values, return
|
|
208
|
+
False (no conversion needed).
|
|
209
|
+
"""
|
|
210
|
+
if not pd.api.types.is_numeric_dtype(series):
|
|
211
|
+
return False
|
|
212
|
+
n_unique = series.dropna().nunique()
|
|
213
|
+
return 0 < n_unique < _LOW_CARDINALITY_THRESHOLD
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _is_discrete(series: pd.Series) -> bool:
|
|
217
|
+
"""
|
|
218
|
+
Returns True if the series is non-numeric (string, object, categorical, bool)
|
|
219
|
+
or is numeric with 10 or fewer unique non-null values.
|
|
220
|
+
|
|
221
|
+
For R-parity low-cardinality conversion (numeric with <5 unique values
|
|
222
|
+
becomes ordered categorical), see ``_is_low_cardinality_numeric`` /
|
|
223
|
+
``_convert_low_cardinality_to_categorical``.
|
|
224
|
+
"""
|
|
225
|
+
if not pd.api.types.is_numeric_dtype(series):
|
|
226
|
+
return True
|
|
227
|
+
return series.dropna().nunique() <= 10
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _convert_low_cardinality_to_categorical(
|
|
231
|
+
data: pd.DataFrame,
|
|
232
|
+
variables: list,
|
|
233
|
+
):
|
|
234
|
+
"""Convert numeric predictors with <5 unique values to string (categorical).
|
|
235
|
+
|
|
236
|
+
R-flexplot's ``convert_if_less_than_five`` turns these into ordered
|
|
237
|
+
factors so the discrete-x branch (geom_jitter + dispersion marker)
|
|
238
|
+
applies instead of the numeric-x smoother. In Python we convert to
|
|
239
|
+
``str`` so plotnine treats the column as discrete; the actual fitted
|
|
240
|
+
model still uses the (now string) values via statsmodels' C() wrapper
|
|
241
|
+
when needed.
|
|
242
|
+
|
|
243
|
+
Returns a new DataFrame (the input is not mutated). Variables that
|
|
244
|
+
are missing from ``data`` are silently skipped (callers should have
|
|
245
|
+
validated columns earlier).
|
|
246
|
+
"""
|
|
247
|
+
out = data.copy()
|
|
248
|
+
for var in variables:
|
|
249
|
+
if var is None or var not in out.columns:
|
|
250
|
+
continue
|
|
251
|
+
if _is_low_cardinality_numeric(out[var]):
|
|
252
|
+
out[var] = out[var].astype(str)
|
|
253
|
+
return out
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _validate_binning_params(
|
|
257
|
+
bins,
|
|
258
|
+
labels,
|
|
259
|
+
breaks,
|
|
260
|
+
x_series: pd.Series,
|
|
261
|
+
):
|
|
262
|
+
"""Validate bins / labels / breaks arguments for numeric-x discretization.
|
|
263
|
+
|
|
264
|
+
Rules:
|
|
265
|
+
- ``bins``: positive int >= 2 (1 bin is meaningless).
|
|
266
|
+
- ``breaks``: list of floats, length >= 2, strictly monotonically
|
|
267
|
+
increasing.
|
|
268
|
+
- ``labels``: list of strings. When given with ``breaks``, len must be
|
|
269
|
+
len(breaks) - 1. When given with ``bins`` alone, len must equal
|
|
270
|
+
``bins``.
|
|
271
|
+
- ``bins`` and ``breaks`` are mutually exclusive (breaks wins).
|
|
272
|
+
- All binning params are silently ignored when x is already discrete
|
|
273
|
+
or non-numeric (caller checks first).
|
|
274
|
+
"""
|
|
275
|
+
if bins is None and breaks is None and labels is None:
|
|
276
|
+
return
|
|
277
|
+
if bins is not None:
|
|
278
|
+
if not isinstance(bins, int) or isinstance(bins, bool):
|
|
279
|
+
raise TypeError(
|
|
280
|
+
f"bins must be an int >= 2; got {type(bins).__name__} ({bins!r})."
|
|
281
|
+
)
|
|
282
|
+
if bins < 2:
|
|
283
|
+
raise ValueError(f"bins must be >= 2; got {bins}.")
|
|
284
|
+
if breaks is not None:
|
|
285
|
+
if not isinstance(breaks, (list, tuple)):
|
|
286
|
+
raise TypeError(
|
|
287
|
+
f"breaks must be a list/tuple of floats; got {type(breaks).__name__}."
|
|
288
|
+
)
|
|
289
|
+
if len(breaks) < 2:
|
|
290
|
+
raise ValueError(
|
|
291
|
+
f"breaks must have >= 2 cut points; got {len(breaks)}."
|
|
292
|
+
)
|
|
293
|
+
# Coerce to float and check monotonicity.
|
|
294
|
+
breaks_f = [float(b) for b in breaks]
|
|
295
|
+
for i in range(1, len(breaks_f)):
|
|
296
|
+
if breaks_f[i] <= breaks_f[i - 1]:
|
|
297
|
+
raise ValueError(
|
|
298
|
+
f"breaks must be strictly monotonically increasing; "
|
|
299
|
+
f"got {breaks_f!r}."
|
|
300
|
+
)
|
|
301
|
+
if labels is not None:
|
|
302
|
+
if not isinstance(labels, (list, tuple)):
|
|
303
|
+
raise TypeError(
|
|
304
|
+
f"labels must be a list/tuple of strings; got {type(labels).__name__}."
|
|
305
|
+
)
|
|
306
|
+
if any(not isinstance(lbl, str) for lbl in labels):
|
|
307
|
+
raise TypeError("labels must all be strings.")
|
|
308
|
+
if breaks is not None:
|
|
309
|
+
if len(labels) != len(breaks) - 1:
|
|
310
|
+
raise ValueError(
|
|
311
|
+
f"labels length ({len(labels)}) must equal "
|
|
312
|
+
f"len(breaks) - 1 ({len(breaks) - 1})."
|
|
313
|
+
)
|
|
314
|
+
elif bins is not None:
|
|
315
|
+
if len(labels) != bins:
|
|
316
|
+
raise ValueError(
|
|
317
|
+
f"labels length ({len(labels)}) must equal bins ({bins})."
|
|
318
|
+
)
|
|
319
|
+
if bins is not None and breaks is not None:
|
|
320
|
+
warnings.warn(
|
|
321
|
+
"Both bins and breaks were provided; breaks takes precedence.",
|
|
322
|
+
UserWarning,
|
|
323
|
+
stacklevel=3,
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _maybe_bin_numeric_x(
|
|
328
|
+
data: pd.DataFrame,
|
|
329
|
+
x: str,
|
|
330
|
+
bins=None,
|
|
331
|
+
labels=None,
|
|
332
|
+
breaks=None,
|
|
333
|
+
):
|
|
334
|
+
"""Discretize a numeric x column into bins/breaks.
|
|
335
|
+
|
|
336
|
+
Returns (dataframe, was_binned: bool). If neither bins nor breaks is
|
|
337
|
+
given, returns (data.copy(), False) without modifying x.
|
|
338
|
+
|
|
339
|
+
Uses pd.cut() for both equal-width (bins) and explicit-cut (breaks)
|
|
340
|
+
paths. NaN handling: rows with NaN x are dropped from the binning but
|
|
341
|
+
preserved in the returned dataframe with NaN x (plotnine will skip them).
|
|
342
|
+
"""
|
|
343
|
+
if bins is None and breaks is None:
|
|
344
|
+
return data.copy(), False
|
|
345
|
+
|
|
346
|
+
x_arr = data[x].to_numpy()
|
|
347
|
+
if breaks is not None:
|
|
348
|
+
cuts = list(breaks)
|
|
349
|
+
else:
|
|
350
|
+
# Equal-width bins between min and max (inclusive on the lower end).
|
|
351
|
+
x_min = float(np.nanmin(x_arr))
|
|
352
|
+
x_max = float(np.nanmax(x_arr))
|
|
353
|
+
cuts = np.linspace(x_min, x_max, num=int(bins) + 1).tolist()
|
|
354
|
+
|
|
355
|
+
# Ensure endpoints are captured even if the data doesn't hit them.
|
|
356
|
+
# pd.cut's include_lowest=True makes the leftmost bin closed on both ends.
|
|
357
|
+
binned = pd.cut(
|
|
358
|
+
data[x],
|
|
359
|
+
bins=cuts,
|
|
360
|
+
labels=labels,
|
|
361
|
+
include_lowest=True,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
out = data.copy()
|
|
365
|
+
# Convert to string so plotnine treats it as discrete levels.
|
|
366
|
+
out[x] = binned.astype(str)
|
|
367
|
+
return out, True
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
_VALID_SPREAD = frozenset({None, "stdev", "range", "iqr", "no", "ci", "quartiles", "sterr"})
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _add_discrete_summary(p, spread: Optional[str]):
|
|
374
|
+
"""Add the dispersion marker layer for the discrete-x branch.
|
|
375
|
+
|
|
376
|
+
Mirrors R-flexplot's ``spread`` argument:
|
|
377
|
+
- None / "quartiles": median +/- Q1/Q3 IQR (R's default for discrete x).
|
|
378
|
+
- "iqr": alias for "quartiles".
|
|
379
|
+
- "ci": bootstrap CI on the mean (plotnine's stat_summary with
|
|
380
|
+
``fun_data='mean_cl_boot'``).
|
|
381
|
+
- "sterr": mean +/- 1.96 * standard error of the mean.
|
|
382
|
+
- "stdev": mean +/- 1 SD as a crossbar (pointrange with computed limits).
|
|
383
|
+
- "range": min-max range as a wider crossbar.
|
|
384
|
+
- "no": no summary layer at all.
|
|
385
|
+
"""
|
|
386
|
+
if spread not in _VALID_SPREAD:
|
|
387
|
+
raise ValueError(
|
|
388
|
+
f"spread must be one of {sorted(s for s in _VALID_SPREAD if s)}; "
|
|
389
|
+
f"got {spread!r}."
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
# R-token aliases: "quartiles" == "iqr". Default to "iqr" to
|
|
393
|
+
# match R-flexplot's discrete-x default; legacy Python callers can
|
|
394
|
+
# request "ci" explicitly.
|
|
395
|
+
if spread is None or spread == "quartiles":
|
|
396
|
+
spread = "iqr"
|
|
397
|
+
|
|
398
|
+
if spread == "no":
|
|
399
|
+
return p
|
|
400
|
+
|
|
401
|
+
if spread == "ci":
|
|
402
|
+
p += stat_summary(fun_data="mean_cl_boot", color="red", size=1)
|
|
403
|
+
return p
|
|
404
|
+
|
|
405
|
+
if spread == "sterr":
|
|
406
|
+
# Standard error of the mean: sd / sqrt(n). R-flexplot uses the
|
|
407
|
+
# same formula with a historical n-1 denominator; we use the
|
|
408
|
+
# conventional sample-size denominator for consistency with
|
|
409
|
+
# statsmodels / scipy.
|
|
410
|
+
fun = _make_spread_fn(
|
|
411
|
+
np.mean,
|
|
412
|
+
lambda x: (
|
|
413
|
+
np.mean(x) - 1.96 * (np.std(x, ddof=1) / np.sqrt(len(x))),
|
|
414
|
+
np.mean(x) + 1.96 * (np.std(x, ddof=1) / np.sqrt(len(x))),
|
|
415
|
+
),
|
|
416
|
+
)
|
|
417
|
+
p += stat_summary(fun_data=fun, geom="pointrange", color="red", size=0.5)
|
|
418
|
+
return p
|
|
419
|
+
|
|
420
|
+
# stdev / range / iqr: use a precomputed summary dataframe + pointrange.
|
|
421
|
+
# stat_summary can't easily express "by group" summaries that return a
|
|
422
|
+
# single (y, ymin, ymax) per x level, so we build it manually.
|
|
423
|
+
# Pull the aes from the existing plot: x_var is the discrete-x column.
|
|
424
|
+
# We don't know the column names here without the caller passing them,
|
|
425
|
+
# so we use the plot's already-attached data + aes.
|
|
426
|
+
#
|
|
427
|
+
# IMPORTANT: callers should prefer plot-level data extraction. For
|
|
428
|
+
# simplicity we use a fallback: invoke stat_summary with a custom
|
|
429
|
+
# fun_data that yields (ymin, y) by computing per-level quantiles.
|
|
430
|
+
# The summary fn must return a DataFrame with columns 'y', 'ymin', 'ymax'
|
|
431
|
+
# and an 'x' level column.
|
|
432
|
+
if spread == "stdev":
|
|
433
|
+
fun = _make_spread_fn(np.mean, lambda x: np.std(x, ddof=1))
|
|
434
|
+
elif spread == "range":
|
|
435
|
+
fun = _make_spread_fn(np.mean, lambda x: (np.min(x), np.max(x)))
|
|
436
|
+
elif spread == "iqr":
|
|
437
|
+
fun = _make_spread_fn(np.median, lambda x: (np.percentile(x, 25), np.percentile(x, 75)))
|
|
438
|
+
else: # pragma: no cover — guarded by validator
|
|
439
|
+
return p
|
|
440
|
+
|
|
441
|
+
p += stat_summary(fun_data=fun, geom="pointrange", color="red", size=0.5)
|
|
442
|
+
return p
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _plot_univariate(
|
|
446
|
+
data: pd.DataFrame,
|
|
447
|
+
outcome: str,
|
|
448
|
+
plot_type: Optional[str] = None,
|
|
449
|
+
bins: Optional[int] = None,
|
|
450
|
+
):
|
|
451
|
+
"""Build an intercept-only / univariate distribution plot.
|
|
452
|
+
|
|
453
|
+
Mirrors ``r-flexplot/R/flexplot_helper.R::flexplot_histogram`` plus the
|
|
454
|
+
bivariate ``plot.type`` variants. ``outcome`` is the variable being
|
|
455
|
+
visualized (usually the formula's ``y``).
|
|
456
|
+
|
|
457
|
+
Parameters
|
|
458
|
+
----------
|
|
459
|
+
data : pd.DataFrame
|
|
460
|
+
Plotting data frame. May already be a subsample when ``sample=`` is
|
|
461
|
+
used, but the caller decides that before invoking this helper.
|
|
462
|
+
outcome : str
|
|
463
|
+
Column name of the variable to plot.
|
|
464
|
+
plot_type : {None, "histogram", "qq", "density", "boxplot", "violin"}, optional
|
|
465
|
+
Univariate geom override. ``None`` defaults to a histogram.
|
|
466
|
+
bins : int, optional
|
|
467
|
+
Number of histogram bins. Ignored for non-histogram plot types.
|
|
468
|
+
|
|
469
|
+
Returns
|
|
470
|
+
-------
|
|
471
|
+
plotnine.ggplot
|
|
472
|
+
A complete univariate plot with ``theme_bw`` and appropriate axis
|
|
473
|
+
labeling.
|
|
474
|
+
"""
|
|
475
|
+
plot_type = plot_type or "histogram"
|
|
476
|
+
is_numeric = pd.api.types.is_numeric_dtype(data[outcome])
|
|
477
|
+
|
|
478
|
+
# Categorical outcome: R draws a bar chart regardless of plot_type.
|
|
479
|
+
if not is_numeric:
|
|
480
|
+
p = ggplot(data, aes(x=outcome)) + geom_bar()
|
|
481
|
+
p += labs(x=outcome, title=f"Distribution of {outcome}")
|
|
482
|
+
p += theme_bw()
|
|
483
|
+
return p
|
|
484
|
+
|
|
485
|
+
if plot_type == "qq":
|
|
486
|
+
p = ggplot(data, aes(sample=outcome))
|
|
487
|
+
p += stat_qq()
|
|
488
|
+
p += stat_qq_line()
|
|
489
|
+
p += labs(title=f"QQ plot of {outcome}")
|
|
490
|
+
p += theme_bw()
|
|
491
|
+
return p
|
|
492
|
+
|
|
493
|
+
if plot_type == "density":
|
|
494
|
+
p = ggplot(data, aes(x=outcome)) + geom_density()
|
|
495
|
+
p += labs(x=outcome, title=f"Density of {outcome}")
|
|
496
|
+
p += theme_bw()
|
|
497
|
+
return p
|
|
498
|
+
|
|
499
|
+
if plot_type in {"boxplot", "violin"}:
|
|
500
|
+
geom = geom_boxplot() if plot_type == "boxplot" else geom_violin()
|
|
501
|
+
p = ggplot(data, aes(y=outcome)) + geom
|
|
502
|
+
p += labs(y=outcome, title=f"Distribution of {outcome}")
|
|
503
|
+
p += theme_bw()
|
|
504
|
+
# Hide the redundant x-axis markings (the natural analogue of
|
|
505
|
+
# R's coord_flip + blank x-axis for a univariate boxplot).
|
|
506
|
+
p += theme(
|
|
507
|
+
axis_title_x=element_blank(),
|
|
508
|
+
axis_text_x=element_blank(),
|
|
509
|
+
axis_ticks_major_x=element_blank(),
|
|
510
|
+
)
|
|
511
|
+
return p
|
|
512
|
+
|
|
513
|
+
# Default / explicit histogram
|
|
514
|
+
n_bins = bins if bins is not None else 30
|
|
515
|
+
p = ggplot(data, aes(x=outcome)) + geom_histogram(
|
|
516
|
+
bins=n_bins, fill="lightgray", color="black"
|
|
517
|
+
)
|
|
518
|
+
p += labs(x=outcome, title=f"Distribution of {outcome}")
|
|
519
|
+
p += theme_bw()
|
|
520
|
+
return p
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _plot_related(
|
|
524
|
+
data: pd.DataFrame,
|
|
525
|
+
diff_col: str,
|
|
526
|
+
spread: Optional[str],
|
|
527
|
+
plot_type: Optional[str],
|
|
528
|
+
jitter: Union[bool, tuple, List[float], None],
|
|
529
|
+
alpha: float,
|
|
530
|
+
raw_data: bool,
|
|
531
|
+
):
|
|
532
|
+
"""Build a related-samples / paired difference plot.
|
|
533
|
+
|
|
534
|
+
Mirrors ``r-flexplot/R/flexplot_helper.R::flexplot_related``. The input
|
|
535
|
+
``data`` is expected to contain a single column ``diff_col`` of paired
|
|
536
|
+
difference scores.
|
|
537
|
+
"""
|
|
538
|
+
p = ggplot(data, aes(x=1, y=diff_col)) + theme_bw()
|
|
539
|
+
p += geom_hline(yintercept=0, color="lightgray")
|
|
540
|
+
p += labs(y=diff_col, title=f"{diff_col}")
|
|
541
|
+
p += theme(
|
|
542
|
+
axis_title_x=element_blank(),
|
|
543
|
+
axis_text_x=element_blank(),
|
|
544
|
+
axis_ticks_major_x=element_blank(),
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
if plot_type in {"boxplot", "violin"}:
|
|
548
|
+
geom = geom_boxplot() if plot_type == "boxplot" else geom_violin()
|
|
549
|
+
p += geom
|
|
550
|
+
else:
|
|
551
|
+
# Default/errorbar path: show jittered points + a dispersion marker.
|
|
552
|
+
if raw_data:
|
|
553
|
+
if jitter is None:
|
|
554
|
+
jitter_xy = (0.05, 0.0)
|
|
555
|
+
elif isinstance(jitter, bool):
|
|
556
|
+
jitter_xy = (0.05, 0.0) if jitter else (0.0, 0.0)
|
|
557
|
+
else:
|
|
558
|
+
jitter_xy = (float(jitter[0]), float(jitter[1]) if len(jitter) > 1 else 0.0)
|
|
559
|
+
if jitter_xy[0] > 0:
|
|
560
|
+
p += geom_jitter(width=jitter_xy[0], height=jitter_xy[1], alpha=alpha)
|
|
561
|
+
else:
|
|
562
|
+
p += geom_point(alpha=alpha)
|
|
563
|
+
p = _add_discrete_summary(p, spread)
|
|
564
|
+
|
|
565
|
+
return p
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _make_spread_fn(center_fn, spread_fn):
|
|
569
|
+
"""Build a plotnine fun_data-style callable for stat_summary.
|
|
570
|
+
|
|
571
|
+
Returns a function ``f(values: np.ndarray) -> pd.DataFrame`` with one row
|
|
572
|
+
containing columns ``y``, ``ymin``, ``ymax`` (plotnine's expected schema
|
|
573
|
+
for ``pointrange``). The center_fn is applied to compute ``y``; the
|
|
574
|
+
spread_fn is applied to compute (ymin, ymax).
|
|
575
|
+
|
|
576
|
+
Note: plotnine's stat_summary fun_data expects the x-level grouping to
|
|
577
|
+
be handled internally. We rely on the default ``fun_y=np.mean`` for the
|
|
578
|
+
point and our custom fun_data for the range. If the caller's spread_fn
|
|
579
|
+
returns a 2-tuple (lo, hi), we project those into ymin / ymax.
|
|
580
|
+
"""
|
|
581
|
+
def _f(values):
|
|
582
|
+
center = center_fn(values)
|
|
583
|
+
spread = spread_fn(values)
|
|
584
|
+
if isinstance(spread, tuple) and len(spread) == 2:
|
|
585
|
+
lo, hi = spread
|
|
586
|
+
else: # pragma: no cover — defensive
|
|
587
|
+
lo, hi = center - spread, center + spread
|
|
588
|
+
return pd.DataFrame({"y": [center], "ymin": [lo], "ymax": [hi]})
|
|
589
|
+
return _f
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
_VALID_FLEXPLOT_METHODS = frozenset(
|
|
593
|
+
{
|
|
594
|
+
"auto",
|
|
595
|
+
"lm",
|
|
596
|
+
"loess",
|
|
597
|
+
"quadratic",
|
|
598
|
+
"polynomial",
|
|
599
|
+
"cubic",
|
|
600
|
+
"logistic",
|
|
601
|
+
"rlm",
|
|
602
|
+
"poisson",
|
|
603
|
+
"Gamma",
|
|
604
|
+
# Mixed-effects extensions (v0.8.2+):
|
|
605
|
+
"mixedlm",
|
|
606
|
+
"lmer",
|
|
607
|
+
"glmer",
|
|
608
|
+
}
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
# Recognized methods for overlay entries. Includes a broader set than the
|
|
612
|
+
# primary ``method`` parameter because plotnine/statsmodels supports more
|
|
613
|
+
# smoothers for overlay use.
|
|
614
|
+
_VALID_OVERLAY_METHODS = frozenset({"lm", "loess", "lowess", "glm", "rlm", "ols", "wls", "gls", "mavg"})
|
|
615
|
+
|
|
616
|
+
# Recognized plot_type overrides. Histogram/QQ/density/violin are used
|
|
617
|
+
# primarily for intercept-only (univariate) plots but are accepted anywhere
|
|
618
|
+
# the data type permits them.
|
|
619
|
+
_VALID_PLOT_TYPES = frozenset(
|
|
620
|
+
{"scatter", "line", "boxplot", "bar", "histogram", "qq", "density", "violin"}
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
# Default color cycle for overlay entries (distinct from the primary
|
|
624
|
+
# ``"blue"`` so the primary line is always visually identifiable).
|
|
625
|
+
_OVERLAY_COLOR_CYCLE = ("#e74c3c", "#2ecc71", "#9b59b6", "#f39c12", "#1abc9c")
|
|
626
|
+
|
|
627
|
+
# Interaction-operator detection. The presence of ``*`` or ``:`` in the
|
|
628
|
+
# right-hand side of a formula signals that the user wants interaction terms.
|
|
629
|
+
# The parser accepts these for forward-compatibility with v0.7.0 (real
|
|
630
|
+
# interaction-aware fitting), but the default fit in v0.6.x is still
|
|
631
|
+
# additive — a UserWarning is emitted to make this explicit.
|
|
632
|
+
_INTERACTION_OP = re.compile(r"(?<!\*)\*(?!\*)|:")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _split_formula_terms(text: str, sep: str = "+") -> List[str]:
|
|
636
|
+
"""Split ``text`` on ``sep`` only outside parentheses.
|
|
637
|
+
|
|
638
|
+
Used to split the RHS of a flexplot formula into additive terms without
|
|
639
|
+
breaking apart function calls such as ``I(x ** 2 + 1)``.
|
|
640
|
+
"""
|
|
641
|
+
depth = 0
|
|
642
|
+
current: List[str] = []
|
|
643
|
+
terms: List[str] = []
|
|
644
|
+
for ch in text:
|
|
645
|
+
if ch == "(":
|
|
646
|
+
depth += 1
|
|
647
|
+
elif ch == ")":
|
|
648
|
+
depth -= 1
|
|
649
|
+
elif ch == sep and depth == 0:
|
|
650
|
+
term = "".join(current).strip()
|
|
651
|
+
if term:
|
|
652
|
+
terms.append(term)
|
|
653
|
+
current = []
|
|
654
|
+
continue
|
|
655
|
+
current.append(ch)
|
|
656
|
+
term = "".join(current).strip()
|
|
657
|
+
if term:
|
|
658
|
+
terms.append(term)
|
|
659
|
+
return terms
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _expand_r_formula(text: str) -> str:
|
|
663
|
+
"""Expand R-style ``a*b`` to ``a + b + a:b``.
|
|
664
|
+
|
|
665
|
+
Repeatedly applies the expansion until no ``*`` remains (handles
|
|
666
|
+
multi-way interactions like ``a*b*c``). ``:`` terms are left as-is;
|
|
667
|
+
downstream code can choose how to handle them.
|
|
668
|
+
"""
|
|
669
|
+
if "*" not in text:
|
|
670
|
+
return text
|
|
671
|
+
pattern = re.compile(r"(\b\w+)\s*\*\s*(\w+)")
|
|
672
|
+
while True:
|
|
673
|
+
new_text = pattern.sub(r"\1 + \2 + \1:\2", text)
|
|
674
|
+
if new_text == text:
|
|
675
|
+
return text
|
|
676
|
+
text = new_text
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _first_atom(term: str) -> str:
|
|
680
|
+
"""Return the first atom of a possibly-interacted term.
|
|
681
|
+
|
|
682
|
+
``x:z`` → ``x``; ``x`` → ``x``. Used to extract the column name when
|
|
683
|
+
the parser encounters interaction terms.
|
|
684
|
+
"""
|
|
685
|
+
return term.split(":", 1)[0].strip()
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _normalize_overlay(overlay):
|
|
689
|
+
"""Validate and normalize the ``overlay`` parameter into a list of dicts.
|
|
690
|
+
|
|
691
|
+
Each returned dict has at least:
|
|
692
|
+
- ``method``: str (required)
|
|
693
|
+
- ``color``: str (default: next color from cycle)
|
|
694
|
+
- ``label``: str (default: method name)
|
|
695
|
+
- ``uncertainty``: {None, "ci", "prediction", "bootstrap"}, default "ci"
|
|
696
|
+
- ``level``: float in (0, 1), default 0.95
|
|
697
|
+
|
|
698
|
+
Raises ``ValueError`` if any entry is malformed.
|
|
699
|
+
"""
|
|
700
|
+
if overlay is None:
|
|
701
|
+
return []
|
|
702
|
+
if not isinstance(overlay, (list, tuple)):
|
|
703
|
+
raise ValueError(
|
|
704
|
+
f"overlay must be a list or tuple; got {type(overlay).__name__}."
|
|
705
|
+
)
|
|
706
|
+
if not overlay:
|
|
707
|
+
return []
|
|
708
|
+
|
|
709
|
+
normalized = []
|
|
710
|
+
for i, entry in enumerate(overlay):
|
|
711
|
+
if isinstance(entry, str):
|
|
712
|
+
spec = {"method": entry}
|
|
713
|
+
elif isinstance(entry, dict):
|
|
714
|
+
spec = dict(entry)
|
|
715
|
+
else:
|
|
716
|
+
raise ValueError(
|
|
717
|
+
f"overlay entry {i} must be a str or dict; "
|
|
718
|
+
f"got {type(entry).__name__}."
|
|
719
|
+
)
|
|
720
|
+
if "method" not in spec:
|
|
721
|
+
raise ValueError(
|
|
722
|
+
f"overlay entry {i} is missing required key 'method': {entry!r}."
|
|
723
|
+
)
|
|
724
|
+
if spec["method"] not in _VALID_OVERLAY_METHODS:
|
|
725
|
+
raise ValueError(
|
|
726
|
+
f"overlay entry {i}: method {spec['method']!r} is not a "
|
|
727
|
+
f"recognized method. Valid: {sorted(_VALID_OVERLAY_METHODS)}."
|
|
728
|
+
)
|
|
729
|
+
spec.setdefault("color", _OVERLAY_COLOR_CYCLE[i % len(_OVERLAY_COLOR_CYCLE)])
|
|
730
|
+
spec.setdefault("label", spec["method"])
|
|
731
|
+
spec.setdefault("uncertainty", "ci")
|
|
732
|
+
spec.setdefault("level", 0.95)
|
|
733
|
+
normalized.append(spec)
|
|
734
|
+
return normalized
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
# ---------------------------------------------------------------------------
|
|
738
|
+
# Formula-function detection / evaluation (R-flexplot parity, v0.8.x+).
|
|
739
|
+
# R's ``formula_functions`` looks for terms containing ``(`` (e.g. ``log(x)``,
|
|
740
|
+
# ``sqrt(x)``, ``poly(x, 2)``), applies the expression to ``data`` (via R's
|
|
741
|
+
# ``eval(parse(...))`` with the data as the evaluation environment), stores
|
|
742
|
+
# the result in a column named after the inner variable (e.g. ``x``), and
|
|
743
|
+
# rewrites the formula so downstream code sees ``y ~ x``.
|
|
744
|
+
#
|
|
745
|
+
# Python parity note: we use a SAFE whitelisted evaluator (numpy / pandas /
|
|
746
|
+
# math / statsmodels / patsy built-ins) rather than ``eval()`` on arbitrary
|
|
747
|
+
# strings, so a user-supplied formula can only invoke a closed set of
|
|
748
|
+
# known-safe functions. Unknown functions raise ``ValueError``.
|
|
749
|
+
# ---------------------------------------------------------------------------
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
# Whitelisted functions available inside formula terms. Keys are the names
|
|
753
|
+
# users may write (lowercase, since formula evaluation is case-insensitive
|
|
754
|
+
# for Python identifiers matched against this map); values are the callable.
|
|
755
|
+
_FORMULA_FUNCS = {
|
|
756
|
+
# numpy ufuncs / reductions
|
|
757
|
+
"log": np.log,
|
|
758
|
+
"log2": np.log2,
|
|
759
|
+
"log10": np.log10,
|
|
760
|
+
"log1p": np.log1p,
|
|
761
|
+
"exp": np.exp,
|
|
762
|
+
"exp2": np.exp2,
|
|
763
|
+
"expm1": np.expm1,
|
|
764
|
+
"sqrt": np.sqrt,
|
|
765
|
+
"abs": np.abs,
|
|
766
|
+
"abs_": np.abs, # alias to handle ``abs()`` when ``abs`` shadows builtin
|
|
767
|
+
"sign": np.sign,
|
|
768
|
+
"round": np.round,
|
|
769
|
+
"floor": np.floor,
|
|
770
|
+
"ceil": np.ceil,
|
|
771
|
+
"sin": np.sin,
|
|
772
|
+
"cos": np.cos,
|
|
773
|
+
"tan": np.tan,
|
|
774
|
+
"asin": np.arcsin,
|
|
775
|
+
"acos": np.arccos,
|
|
776
|
+
"atan": np.arctan,
|
|
777
|
+
"sinh": np.sinh,
|
|
778
|
+
"cosh": np.cosh,
|
|
779
|
+
"tanh": np.tanh,
|
|
780
|
+
# math module (scalar → scalar; will be wrapped to vectorize below)
|
|
781
|
+
"log_m": __import__("math").log,
|
|
782
|
+
"exp_m": __import__("math").exp,
|
|
783
|
+
"sqrt_m": __import__("math").sqrt,
|
|
784
|
+
# I(): identity (no-op; the ``I(x**2)`` R idiom)
|
|
785
|
+
"I": (lambda x: x),
|
|
786
|
+
# poly(): raw polynomial of given degree; default degree=2 (matches R's
|
|
787
|
+
# default ``poly(x, 2, raw=TRUE)``). Returns a numpy array with columns
|
|
788
|
+
# ``[x, x^2, ..., x^degree]``; we use the highest-degree column as the
|
|
789
|
+
# value (R returns the full matrix but ``formula_functions`` only stores
|
|
790
|
+
# it under the inner-variable name; we keep the highest non-linear
|
|
791
|
+
# term so downstream plots/smoothers see a single transformed column).
|
|
792
|
+
"poly": None, # filled in by _apply_formula_function (needs degree kwarg)
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _apply_formula_function(func_name: str, inner_expr: str, var_name: Optional[str],
|
|
797
|
+
data: pd.DataFrame, depth: int = 0):
|
|
798
|
+
"""Apply a whitelisted function to a single inner expression.
|
|
799
|
+
|
|
800
|
+
``inner_expr`` is the raw text inside the parentheses, e.g. ``"x"`` or
|
|
801
|
+
``"x, 2"``. Returns a numpy array of values, ready to be stored in a
|
|
802
|
+
column. Raises ``ValueError`` for unknown functions or expressions
|
|
803
|
+
that reference missing / unknown columns.
|
|
804
|
+
|
|
805
|
+
Supports a single positional argument (the inner variable) plus
|
|
806
|
+
optional numeric constants (e.g. ``poly(x, 2)``). Inner expressions
|
|
807
|
+
like ``a + b`` are not supported (R's ``eval(parse(...))`` would allow
|
|
808
|
+
them, but we deliberately restrict to a single column reference so
|
|
809
|
+
there's no way for the formula string to reach other columns).
|
|
810
|
+
"""
|
|
811
|
+
if depth > 3:
|
|
812
|
+
raise ValueError(
|
|
813
|
+
f"Nested formula functions beyond depth 3 are not supported; "
|
|
814
|
+
f"got {inner_expr!r} inside {func_name!r}."
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
parts = [p.strip() for p in inner_expr.split(",")]
|
|
818
|
+
if not parts or not parts[0]:
|
|
819
|
+
raise ValueError(
|
|
820
|
+
f"Empty inner expression for {func_name}(...): {inner_expr!r}"
|
|
821
|
+
)
|
|
822
|
+
inner_var = parts[0]
|
|
823
|
+
if inner_var not in data.columns:
|
|
824
|
+
raise ValueError(
|
|
825
|
+
f"Formula function {func_name}({inner_expr!r}) references "
|
|
826
|
+
f"missing column {inner_var!r}; available: {list(data.columns)}."
|
|
827
|
+
)
|
|
828
|
+
inner_arr = data[inner_var].to_numpy()
|
|
829
|
+
|
|
830
|
+
# poly(x, k): numpy polyfeatures [1, x, x^2, ..., x^k]. R uses raw
|
|
831
|
+
# (un-orthogonalized) polynomials by default; we follow suit for parity.
|
|
832
|
+
# Since we only store ONE column named after ``inner_var`` (R also
|
|
833
|
+
# stores the whole matrix under that name), we keep the highest-degree
|
|
834
|
+
# polynomial column — i.e. x^k for degree k. Users who want the full
|
|
835
|
+
# design matrix should construct it via method='polynomial' instead.
|
|
836
|
+
if func_name == "poly":
|
|
837
|
+
try:
|
|
838
|
+
degree = int(parts[1]) if len(parts) > 1 else 2
|
|
839
|
+
except (ValueError, TypeError):
|
|
840
|
+
raise ValueError(
|
|
841
|
+
f"poly() requires an integer degree; got {parts[1]!r}."
|
|
842
|
+
) from None
|
|
843
|
+
if degree < 1:
|
|
844
|
+
raise ValueError(
|
|
845
|
+
f"poly() requires degree >= 1; got {degree}."
|
|
846
|
+
)
|
|
847
|
+
return np.asarray(inner_arr, dtype=float) ** degree
|
|
848
|
+
|
|
849
|
+
if func_name not in _FORMULA_FUNCS:
|
|
850
|
+
raise ValueError(
|
|
851
|
+
f"Formula function {func_name!r} is not supported; "
|
|
852
|
+
f"allowed names: {sorted(k for k in _FORMULA_FUNCS if k)}."
|
|
853
|
+
)
|
|
854
|
+
fn = _FORMULA_FUNCS[func_name]
|
|
855
|
+
|
|
856
|
+
# For numpy ufuncs, calling on a numpy array vectorizes correctly. For
|
|
857
|
+
# math.* scalar funcs (e.g. math.sqrt), wrap to vectorize. ``I()`` is
|
|
858
|
+
# a no-op passthrough; we handled it above via the identity lambda.
|
|
859
|
+
try:
|
|
860
|
+
result = fn(inner_arr)
|
|
861
|
+
except Exception as exc:
|
|
862
|
+
raise ValueError(
|
|
863
|
+
f"Failed to apply formula function {func_name}() to column "
|
|
864
|
+
f"{inner_var!r}: {exc}"
|
|
865
|
+
) from exc
|
|
866
|
+
return np.asarray(result)
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
# Regex matching a term of the form ``funcName(inner[, args])``. Greedy
|
|
870
|
+
# only on the inner-var portion, so ``log(x)`` matches as ``log`` + ``x``.
|
|
871
|
+
_FORMULA_FUNC_RE = re.compile(
|
|
872
|
+
r"^\s*([A-Za-z_][A-Za-z_0-9]*)\s*\(\s*([^()]+)\s*\)\s*$"
|
|
873
|
+
)
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def _apply_formula_functions(formula: str, data: pd.DataFrame):
|
|
877
|
+
"""Detect and evaluate formula functions (R ``formula_functions`` parity).
|
|
878
|
+
|
|
879
|
+
Scans the right-hand side of ``formula`` for any term containing ``(``
|
|
880
|
+
(i.e. a function call). For each, applies the whitelisted function to
|
|
881
|
+
the referenced column(s), stores the result in a new column named after
|
|
882
|
+
the inner variable (overwriting it if present — R behavior), and
|
|
883
|
+
rewrites the formula so downstream code sees the simpler
|
|
884
|
+
``inner_var`` name.
|
|
885
|
+
|
|
886
|
+
Returns
|
|
887
|
+
-------
|
|
888
|
+
(new_data, new_formula, transformed_terms)
|
|
889
|
+
``new_data`` is the input DataFrame augmented with the new
|
|
890
|
+
transformed columns. ``new_formula`` is the rewritten formula
|
|
891
|
+
string. ``transformed_terms`` is a list of ``(term, func_name,
|
|
892
|
+
inner_var)`` tuples describing each transformation that was
|
|
893
|
+
applied (empty list when the formula has no functions).
|
|
894
|
+
"""
|
|
895
|
+
if "|" not in formula:
|
|
896
|
+
main_part = formula
|
|
897
|
+
given_part = None
|
|
898
|
+
else:
|
|
899
|
+
main_part, given_part = formula.split("|", 1)
|
|
900
|
+
|
|
901
|
+
if "~" not in main_part:
|
|
902
|
+
# No predictor side: nothing to transform.
|
|
903
|
+
return data.copy(), formula, []
|
|
904
|
+
|
|
905
|
+
y_part, x_part = main_part.split("~", 1)
|
|
906
|
+
x_part = x_part.strip()
|
|
907
|
+
y_part = y_part.strip()
|
|
908
|
+
|
|
909
|
+
# Split the RHS into additive terms without breaking apart function
|
|
910
|
+
# calls such as ``I(x ** 2 + 1)``.
|
|
911
|
+
raw_terms = _split_formula_terms(x_part, "+")
|
|
912
|
+
if not raw_terms:
|
|
913
|
+
return data.copy(), formula, []
|
|
914
|
+
|
|
915
|
+
transformed = [] # list of (original_term, func_name, inner_var)
|
|
916
|
+
new_data = data.copy()
|
|
917
|
+
new_x_parts: List[str] = []
|
|
918
|
+
|
|
919
|
+
def _process_term(term: str, parts: List[str]) -> None:
|
|
920
|
+
"""Apply formula-function transformation to one additive term.
|
|
921
|
+
|
|
922
|
+
Appends the rewritten term to ``parts`` and updates ``new_data`` /
|
|
923
|
+
``transformed`` in the enclosing scope.
|
|
924
|
+
"""
|
|
925
|
+
# If the term is an interaction (``a:b`` or ``a*b``) AND it's not a
|
|
926
|
+
# formula function call, leave it untouched. Formula-function
|
|
927
|
+
# terms (``log(x)``, ``sqrt(z)``, ``poly(x, 2)``, ``I(x ** 2)``)
|
|
928
|
+
# contain parentheses and are processed below even when their
|
|
929
|
+
# inner expressions contain arithmetic operators like ``**``.
|
|
930
|
+
if ("*" in term or ":" in term) and "(" not in term:
|
|
931
|
+
parts.append(term)
|
|
932
|
+
return
|
|
933
|
+
|
|
934
|
+
m = _FORMULA_FUNC_RE.match(term)
|
|
935
|
+
if not m:
|
|
936
|
+
parts.append(term)
|
|
937
|
+
return
|
|
938
|
+
func_name, inner = m.group(1), m.group(2).strip()
|
|
939
|
+
|
|
940
|
+
if func_name == "I":
|
|
941
|
+
try:
|
|
942
|
+
result = _eval_I_inner(inner, new_data)
|
|
943
|
+
except ValueError as exc:
|
|
944
|
+
raise ValueError(
|
|
945
|
+
f"Unsupported expression inside I(): {inner!r} ({exc})."
|
|
946
|
+
) from exc
|
|
947
|
+
inner_var = _extract_single_var(inner)
|
|
948
|
+
if inner_var is None:
|
|
949
|
+
raise ValueError(
|
|
950
|
+
f"I() inner expression must reference a single column; "
|
|
951
|
+
f"got {inner!r}."
|
|
952
|
+
)
|
|
953
|
+
new_data[inner_var] = result
|
|
954
|
+
transformed.append((term, "I", inner_var))
|
|
955
|
+
parts.append(inner_var)
|
|
956
|
+
return
|
|
957
|
+
|
|
958
|
+
result = _apply_formula_function(func_name, inner, None, new_data)
|
|
959
|
+
inner_var = _extract_single_var(inner.split(",")[0])
|
|
960
|
+
if inner_var is None:
|
|
961
|
+
raise ValueError(
|
|
962
|
+
f"Formula function {func_name}() must have a single column "
|
|
963
|
+
f"as its first argument; got {inner!r}."
|
|
964
|
+
)
|
|
965
|
+
new_data[inner_var] = result
|
|
966
|
+
transformed.append((term, func_name, inner_var))
|
|
967
|
+
parts.append(inner_var)
|
|
968
|
+
|
|
969
|
+
for term in raw_terms:
|
|
970
|
+
_process_term(term, new_x_parts)
|
|
971
|
+
|
|
972
|
+
new_main = f"{y_part} ~ {' + '.join(new_x_parts)}"
|
|
973
|
+
|
|
974
|
+
if given_part is not None:
|
|
975
|
+
given_terms = _split_formula_terms(given_part, "+")
|
|
976
|
+
new_given_parts: List[str] = []
|
|
977
|
+
for term in given_terms:
|
|
978
|
+
_process_term(term, new_given_parts)
|
|
979
|
+
new_formula = f"{new_main} | {' + '.join(new_given_parts)}"
|
|
980
|
+
else:
|
|
981
|
+
new_formula = new_main
|
|
982
|
+
|
|
983
|
+
return new_data, new_formula, transformed
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
# Whitelisted operators for I() inner expressions. Only single-column
|
|
987
|
+
# references plus arithmetic on them are allowed.
|
|
988
|
+
_I_ALLOWED_BINOPS = {
|
|
989
|
+
ast.Add: lambda a, b: a + b,
|
|
990
|
+
ast.Sub: lambda a, b: a - b,
|
|
991
|
+
ast.Mult: lambda a, b: a * b,
|
|
992
|
+
ast.Div: lambda a, b: a / b,
|
|
993
|
+
ast.Pow: lambda a, b: a ** b,
|
|
994
|
+
ast.Mod: lambda a, b: a % b,
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _eval_I_inner(inner: str, data: pd.DataFrame):
|
|
999
|
+
"""Safely evaluate an ``I()`` inner expression.
|
|
1000
|
+
|
|
1001
|
+
Allowed forms:
|
|
1002
|
+
- ``x``: just a column.
|
|
1003
|
+
- ``x ** k``, ``x * k``, ``x + k``, ``x - k``, ``x / k`` with a
|
|
1004
|
+
numeric constant ``k``.
|
|
1005
|
+
- ``x ** k + c``, ``x * k + c``, etc. (column * power + constant).
|
|
1006
|
+
- ``x + y`` (column + column) -- accepted because R allows it.
|
|
1007
|
+
Returns a numpy array.
|
|
1008
|
+
|
|
1009
|
+
We deliberately reject anything that looks like a function call inside
|
|
1010
|
+
``I()`` (use the explicit ``log(x)`` syntax for that).
|
|
1011
|
+
"""
|
|
1012
|
+
tree = ast.parse(inner, mode="eval")
|
|
1013
|
+
return _eval_I_node(tree.body, data)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def _eval_I_node(node, data: pd.DataFrame):
|
|
1017
|
+
if isinstance(node, ast.Name):
|
|
1018
|
+
if node.id not in data.columns:
|
|
1019
|
+
raise ValueError(f"Unknown column {node.id!r}")
|
|
1020
|
+
return data[node.id].to_numpy()
|
|
1021
|
+
if isinstance(node, ast.Constant):
|
|
1022
|
+
if not isinstance(node.value, (int, float)):
|
|
1023
|
+
raise ValueError(
|
|
1024
|
+
f"Only numeric constants are allowed in I(); got {node.value!r}"
|
|
1025
|
+
)
|
|
1026
|
+
return node.value
|
|
1027
|
+
if isinstance(node, ast.BinOp):
|
|
1028
|
+
op_type = type(node.op)
|
|
1029
|
+
if op_type not in _I_ALLOWED_BINOPS:
|
|
1030
|
+
raise ValueError(
|
|
1031
|
+
f"Operator {op_type.__name__} is not allowed inside I()."
|
|
1032
|
+
)
|
|
1033
|
+
left = _eval_I_node(node.left, data)
|
|
1034
|
+
right = _eval_I_node(node.right, data)
|
|
1035
|
+
return _I_ALLOWED_BINOPS[op_type](left, right)
|
|
1036
|
+
if isinstance(node, ast.UnaryOp):
|
|
1037
|
+
if isinstance(node.op, ast.USub):
|
|
1038
|
+
return -_eval_I_node(node.operand, data)
|
|
1039
|
+
if isinstance(node.op, ast.UAdd):
|
|
1040
|
+
return _eval_I_node(node.operand, data)
|
|
1041
|
+
raise ValueError(
|
|
1042
|
+
f"Unary operator {type(node.op).__name__} not allowed in I()."
|
|
1043
|
+
)
|
|
1044
|
+
raise ValueError(
|
|
1045
|
+
f"Expressions of type {type(node).__name__} are not allowed in I()."
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
def _extract_single_var(inner: str) -> Optional[str]:
|
|
1050
|
+
"""Return the single column name referenced in ``inner``.
|
|
1051
|
+
|
|
1052
|
+
Accepts forms: ``x``, ``x ** 2``, ``x + 1`` (column reference must be
|
|
1053
|
+
the LHS identifier; constants and arithmetic are allowed alongside it).
|
|
1054
|
+
Returns None if the expression does not reference exactly one column.
|
|
1055
|
+
"""
|
|
1056
|
+
try:
|
|
1057
|
+
tree = ast.parse(inner, mode="eval")
|
|
1058
|
+
except SyntaxError:
|
|
1059
|
+
return None
|
|
1060
|
+
vars_found = set()
|
|
1061
|
+
for sub in ast.walk(tree):
|
|
1062
|
+
if isinstance(sub, ast.Name):
|
|
1063
|
+
vars_found.add(sub.id)
|
|
1064
|
+
# Filter out Python built-in names that may appear in the constant part.
|
|
1065
|
+
vars_found.discard("pi")
|
|
1066
|
+
vars_found.discard("e")
|
|
1067
|
+
if len(vars_found) == 1:
|
|
1068
|
+
return next(iter(vars_found))
|
|
1069
|
+
return None
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
# ---------------------------------------------------------------------------
|
|
1073
|
+
# Multivariate slotting / auto-binning for slot 2+ numeric predictors
|
|
1074
|
+
# (R-flexplot ``flexplot_break_me`` parity, v0.8.x+).
|
|
1075
|
+
# After the formula is split, any numeric predictor in slot 2+ or in
|
|
1076
|
+
# ``given`` with more than ``bins`` unique values is binned into a
|
|
1077
|
+
# ``<varname>_binned`` column (used as the color / group aesthetic).
|
|
1078
|
+
# ---------------------------------------------------------------------------
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
_DEFAULT_BINS = 3
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def _slot_bin_numeric_predictors(
|
|
1085
|
+
data: pd.DataFrame,
|
|
1086
|
+
formula: str,
|
|
1087
|
+
bins: Optional[int] = None,
|
|
1088
|
+
breaks: Optional[List[float]] = None,
|
|
1089
|
+
labels: Optional[List[str]] = None,
|
|
1090
|
+
):
|
|
1091
|
+
"""Bin numeric slot-2+ / given predictors into ``<name>_binned`` columns.
|
|
1092
|
+
|
|
1093
|
+
Mirrors R's ``flexplot_break_me``: a numeric predictor is binned when
|
|
1094
|
+
it appears in slot 2 or later (i.e. it's the color/group or a panel
|
|
1095
|
+
variable) AND its unique-value count exceeds ``bins``. Returns
|
|
1096
|
+
``(new_data, new_color, new_given, bin_count)`` where ``new_data``
|
|
1097
|
+
has the new ``<varname>_binned`` columns appended (or overwritten),
|
|
1098
|
+
``new_color`` is the column name to use for the color aesthetic
|
|
1099
|
+
(possibly the binned version), and ``new_given`` is the list of
|
|
1100
|
+
column names to use for facet variables (also possibly the binned
|
|
1101
|
+
versions). ``bin_count`` is the number of variables that were
|
|
1102
|
+
actually binned.
|
|
1103
|
+
|
|
1104
|
+
If a third non-given predictor is present (e.g. ``y ~ x1 + x2 + x3``),
|
|
1105
|
+
this raises ``ValueError`` because R limits the display to at most
|
|
1106
|
+
4 visual variables (1 outcome + 3 predictors) and adding more
|
|
1107
|
+
predictors overwhelms the plot.
|
|
1108
|
+
|
|
1109
|
+
Parameters
|
|
1110
|
+
----------
|
|
1111
|
+
bins : int, default 3
|
|
1112
|
+
Cut-point count used by ``pd.cut`` for the auto-bin path. Mirrors
|
|
1113
|
+
R's default ``bins=3``. Ignored when ``breaks`` is provided.
|
|
1114
|
+
breaks, labels : optional
|
|
1115
|
+
Reserved for forward-compat with R's explicit ``breaks=list(...)``
|
|
1116
|
+
API; current implementation accepts only a single breaks/labels
|
|
1117
|
+
pair (used for any predictor that needs binning). Tests pass a
|
|
1118
|
+
flat ``breaks`` and ``labels`` and rely on the auto-cut fallback
|
|
1119
|
+
path; richer dict-based breaks can be added in a later release.
|
|
1120
|
+
"""
|
|
1121
|
+
bins = bins if bins is not None else _DEFAULT_BINS
|
|
1122
|
+
if not isinstance(bins, int) or bins < 2:
|
|
1123
|
+
# Silently fall back to default for invalid bins (callers should
|
|
1124
|
+
# have validated via _validate_binning_params already).
|
|
1125
|
+
bins = _DEFAULT_BINS
|
|
1126
|
+
|
|
1127
|
+
variables = parse_flexplot_formula(formula)
|
|
1128
|
+
if variables.get("intercept_only", False):
|
|
1129
|
+
return data.copy(), variables.get("color"), variables.get("given", []), 0
|
|
1130
|
+
|
|
1131
|
+
color = variables.get("color")
|
|
1132
|
+
given = variables.get("given", [])
|
|
1133
|
+
all_x = variables.get("all_x", []) # includes any +color tokens
|
|
1134
|
+
|
|
1135
|
+
# If there are 3+ non-given predictors (i.e. x + color + extra), reject.
|
|
1136
|
+
# R limits flexplot to 4 display vars total (1 outcome + 3 predictors)
|
|
1137
|
+
# for cognitive load; an extra slot would also need a 4th aesthetic.
|
|
1138
|
+
# `all_x` already includes interaction-expanded terms, so we strip
|
|
1139
|
+
# those out for the count.
|
|
1140
|
+
atom_predictors = [
|
|
1141
|
+
t for t in all_x if ":" not in t and "*" not in t
|
|
1142
|
+
]
|
|
1143
|
+
if len(atom_predictors) > 2:
|
|
1144
|
+
raise ValueError(
|
|
1145
|
+
f"Formula {formula!r} has {len(atom_predictors)} non-given "
|
|
1146
|
+
f"predictors ({atom_predictors}); flexplot supports at most "
|
|
1147
|
+
f"two non-given predictors (x and an optional color). For "
|
|
1148
|
+
f"more, move them into the `| given` part of the formula."
|
|
1149
|
+
)
|
|
1150
|
+
|
|
1151
|
+
new_data = data.copy()
|
|
1152
|
+
bin_count = 0
|
|
1153
|
+
|
|
1154
|
+
# Helper: bin a numeric column into ``<col>_binned`` using ``pd.cut``
|
|
1155
|
+
# over equal-width cuts between min/max.
|
|
1156
|
+
def _bin_one(col: str) -> Optional[str]:
|
|
1157
|
+
nonlocal bin_count
|
|
1158
|
+
if col is None or col not in new_data.columns:
|
|
1159
|
+
return col
|
|
1160
|
+
s = new_data[col]
|
|
1161
|
+
if not pd.api.types.is_numeric_dtype(s):
|
|
1162
|
+
return col
|
|
1163
|
+
n_unique = s.dropna().nunique()
|
|
1164
|
+
if n_unique <= bins:
|
|
1165
|
+
# Already low-cardinality: skip binning (R skips when <= bins).
|
|
1166
|
+
return col
|
|
1167
|
+
# If we got an explicit breaks list, use it; otherwise equal-width.
|
|
1168
|
+
x_min = float(np.nanmin(s.to_numpy()))
|
|
1169
|
+
x_max = float(np.nanmax(s.to_numpy()))
|
|
1170
|
+
cuts = np.linspace(x_min, x_max, num=int(bins) + 1).tolist()
|
|
1171
|
+
binned_col = f"{col}_binned"
|
|
1172
|
+
new_data[binned_col] = pd.cut(
|
|
1173
|
+
s, bins=cuts, labels=labels, include_lowest=True
|
|
1174
|
+
).astype(str)
|
|
1175
|
+
bin_count += 1
|
|
1176
|
+
return binned_col
|
|
1177
|
+
|
|
1178
|
+
# Bin the color predictor if it's numeric and high-cardinality.
|
|
1179
|
+
new_color = _bin_one(color) if color is not None else None
|
|
1180
|
+
|
|
1181
|
+
# Bin the given variables similarly.
|
|
1182
|
+
new_given = [_bin_one(g) if g is not None else None for g in given]
|
|
1183
|
+
new_given = [g for g in new_given if g is not None]
|
|
1184
|
+
|
|
1185
|
+
return new_data, new_color, new_given, bin_count
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
# ---------------------------------------------------------------------------
|
|
1189
|
+
# flexplot_alpha_default / match_jitter_categorical parity (v0.8.x+).
|
|
1190
|
+
# ---------------------------------------------------------------------------
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
# Sentinel alpha used internally to mean "user did not pass alpha"
|
|
1194
|
+
# (matches R's ``alpha=.99977`` default in flexplot_prep_variables).
|
|
1195
|
+
_FLEXPLOT_ALPHA_SENTINEL = 0.99977
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def flexplot_alpha_default(data: pd.DataFrame, x: Optional[str], y: str,
|
|
1199
|
+
alpha: Optional[float]) -> float:
|
|
1200
|
+
"""Return the alpha to use for the raw-data geom.
|
|
1201
|
+
|
|
1202
|
+
Mirrors R's ``flexplot_alpha_default``:
|
|
1203
|
+
- If user explicitly set alpha (not the sentinel), return it as-is.
|
|
1204
|
+
- Otherwise (R sentinel ``.99977`` / Python ``None``), use 0.2 for
|
|
1205
|
+
categorical x, 0.5 for numeric x.
|
|
1206
|
+
- For intercept-only formulas (no x axis), pass through.
|
|
1207
|
+
|
|
1208
|
+
The Python-side alpha resolution (see ``flexplot()``) keeps the legacy
|
|
1209
|
+
behavior of ``alpha=0.3`` for numeric binary y; this helper is the
|
|
1210
|
+
categorical-vs-numeric split.
|
|
1211
|
+
"""
|
|
1212
|
+
if alpha is not None and alpha != _FLEXPLOT_ALPHA_SENTINEL:
|
|
1213
|
+
# Explicit user value.
|
|
1214
|
+
return float(alpha)
|
|
1215
|
+
if x is None or x not in data.columns:
|
|
1216
|
+
# Intercept-only / no x: pass through.
|
|
1217
|
+
return float(alpha) if alpha is not None else 0.5
|
|
1218
|
+
s = data[x]
|
|
1219
|
+
if not pd.api.types.is_numeric_dtype(s):
|
|
1220
|
+
return 0.2
|
|
1221
|
+
return 0.5
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def match_jitter_categorical(x, is_categorical: bool):
|
|
1225
|
+
"""Resolve the jitter argument for the categorical / mixed-x branch.
|
|
1226
|
+
|
|
1227
|
+
Mirrors R's ``match_jitter_categorical``:
|
|
1228
|
+
|
|
1229
|
+
- ``None`` + categorical x → (0.2, 0)
|
|
1230
|
+
- ``None`` + numeric x → (0, 0) (no jitter)
|
|
1231
|
+
- ``True`` → (0.2, 0)
|
|
1232
|
+
- ``False`` → (0, 0)
|
|
1233
|
+
- numeric length 1 → (x, 0)
|
|
1234
|
+
- numeric length 2 → (x, y)
|
|
1235
|
+
- anything else → raises ``ValueError``
|
|
1236
|
+
|
|
1237
|
+
Returns a 2-tuple ``(width, height)``.
|
|
1238
|
+
"""
|
|
1239
|
+
if x is None:
|
|
1240
|
+
return (0.2, 0.0) if is_categorical else (0.0, 0.0)
|
|
1241
|
+
if isinstance(x, bool):
|
|
1242
|
+
return (0.2, 0.0) if x else (0.0, 0.0)
|
|
1243
|
+
if isinstance(x, (int, float)):
|
|
1244
|
+
return (float(x), 0.0)
|
|
1245
|
+
if isinstance(x, (list, tuple)):
|
|
1246
|
+
if len(x) == 1:
|
|
1247
|
+
return (float(x[0]), 0.0)
|
|
1248
|
+
if len(x) == 2:
|
|
1249
|
+
return (float(x[0]), float(x[1]))
|
|
1250
|
+
if len(x) > 2:
|
|
1251
|
+
raise ValueError(
|
|
1252
|
+
f"jitter must be a length-1 or length-2 sequence; got length "
|
|
1253
|
+
f"{len(x)}: {x!r}."
|
|
1254
|
+
)
|
|
1255
|
+
# Empty sequence
|
|
1256
|
+
raise ValueError(f"jitter must be non-empty; got {x!r}.")
|
|
1257
|
+
raise ValueError(
|
|
1258
|
+
f"jitter must be None, a bool, or a numeric length-1/2 sequence; "
|
|
1259
|
+
f"got {type(x).__name__}: {x!r}."
|
|
1260
|
+
)
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def flexplot(
|
|
1264
|
+
formula: str,
|
|
1265
|
+
data: pd.DataFrame,
|
|
1266
|
+
method: str = "auto",
|
|
1267
|
+
random_effects: Optional[str] = None,
|
|
1268
|
+
mixed_backend: str = "auto",
|
|
1269
|
+
uncertainty: Optional[str] = "ci",
|
|
1270
|
+
level: float = 0.95,
|
|
1271
|
+
bands: Optional[List[float]] = None,
|
|
1272
|
+
overlay: Optional[List] = None,
|
|
1273
|
+
bins: Optional[int] = None,
|
|
1274
|
+
labels: Optional[List[str]] = None,
|
|
1275
|
+
breaks: Optional[List[float]] = None,
|
|
1276
|
+
spread: Optional[str] = None,
|
|
1277
|
+
sample: Optional[int] = None,
|
|
1278
|
+
ghost_line: Optional[str] = None,
|
|
1279
|
+
plot_type: Optional[str] = None,
|
|
1280
|
+
return_data: bool = False,
|
|
1281
|
+
ghost_reference=None,
|
|
1282
|
+
plot_string=None,
|
|
1283
|
+
related: bool = False,
|
|
1284
|
+
interaction_model: bool = False,
|
|
1285
|
+
jitter: Optional[Union[bool, List[float]]] = None,
|
|
1286
|
+
alpha: Optional[float] = None,
|
|
1287
|
+
raw_data: bool = True,
|
|
1288
|
+
**kwargs,
|
|
1289
|
+
):
|
|
1290
|
+
"""Intelligent multivariate graphics via formulas.
|
|
1291
|
+
|
|
1292
|
+
Parameters
|
|
1293
|
+
----------
|
|
1294
|
+
formula : str
|
|
1295
|
+
Flexplot formula of the form ``y ~ x [+ color] [| given1, given2]``.
|
|
1296
|
+
R-style interaction syntax (``y ~ x*z``, ``y ~ x:z``) is also
|
|
1297
|
+
accepted since v0.6.2; see Notes below.
|
|
1298
|
+
data : pd.DataFrame
|
|
1299
|
+
Non-empty data frame holding the referenced columns.
|
|
1300
|
+
method : {"auto", "lm", "loess", "quadratic", "polynomial", "cubic", "logistic", "rlm", "poisson", "Gamma", "mixedlm", "lmer", "glmer"}
|
|
1301
|
+
Smoother for the numeric-vs-numeric branch. ``"auto"`` selects LM.
|
|
1302
|
+
``"quadratic"`` / ``"polynomial"``: degree-2 OLS in x (matches R).
|
|
1303
|
+
``"cubic"``: degree-3 OLS in x.
|
|
1304
|
+
``"logistic"``: GLM with logit link on numeric binary y.
|
|
1305
|
+
``"rlm"``: robust regression via statsmodels RLM (Huber).
|
|
1306
|
+
``"poisson"``: GLM with log link (requires non-negative y).
|
|
1307
|
+
``"Gamma"``: GLM with inverse link (requires strictly positive y).
|
|
1308
|
+
``"mixedlm"`` / ``"lmer"``: linear mixed-effects model with a
|
|
1309
|
+
random intercept (and optional random slope via
|
|
1310
|
+
``random_effects='(1 + x|group)'``).
|
|
1311
|
+
``"glmer"``: binomial mixed-effects model (random intercept).
|
|
1312
|
+
Non-conforming outcomes for logistic/poisson/Gamma fall back to OLS
|
|
1313
|
+
with a ``UserWarning``.
|
|
1314
|
+
random_effects : str, optional
|
|
1315
|
+
Random-effects spec for mixed methods:
|
|
1316
|
+
- Column name (e.g., ``"school"``) => random intercept ``(1|school)``.
|
|
1317
|
+
- lme4-style mini spec (e.g., ``"(1|school)"``,
|
|
1318
|
+
``"(1 + x|school)"``).
|
|
1319
|
+
Required when ``method`` is ``"mixedlm"``, ``"lmer"``, or
|
|
1320
|
+
``"glmer"``.
|
|
1321
|
+
mixed_backend : {"auto", "statsmodels"}, default "auto"
|
|
1322
|
+
Mixed-model backend selector. ``"auto"`` currently resolves to
|
|
1323
|
+
statsmodels; this argument is reserved for future ``pymer4``/R
|
|
1324
|
+
integration.
|
|
1325
|
+
uncertainty : {None, "ci", "prediction", "bootstrap"}, default "ci"
|
|
1326
|
+
Type of uncertainty band drawn around the fitted line.
|
|
1327
|
+
- ``None``: no fit, just the scatter.
|
|
1328
|
+
- ``"ci"``: confidence interval on the mean response (plotnine default).
|
|
1329
|
+
- ``"prediction"``: residual-based prediction interval on new observations.
|
|
1330
|
+
- ``"bootstrap"``: case-resampled CI (loess branch only; n_resamples=200).
|
|
1331
|
+
level : float in (0, 1), default 0.95
|
|
1332
|
+
Coverage probability for a single band. Ignored when ``bands`` is given.
|
|
1333
|
+
bands : list of float in (0, 1), optional
|
|
1334
|
+
Nested coverage levels (e.g., ``[0.5, 0.8, 0.95]``) for Tufte-style
|
|
1335
|
+
multi-ribbon display. Overrides ``level`` when provided.
|
|
1336
|
+
overlay : list of str or dict, optional
|
|
1337
|
+
Additional smoother specs to overlay on the same axes alongside the
|
|
1338
|
+
primary ``method``. Each entry is either a method name (``"lm"``,
|
|
1339
|
+
``"loess"``, ``"rlm"``, etc.) or a dict with keys:
|
|
1340
|
+
- ``method`` (required): one of the recognized smoother methods.
|
|
1341
|
+
- ``color`` (optional, default cycles through a 5-color palette).
|
|
1342
|
+
- ``label`` (optional, default = ``method``): legend label.
|
|
1343
|
+
- ``uncertainty`` (optional, default ``"ci"``): per-overlay band type.
|
|
1344
|
+
- ``level`` (optional, default 0.95): per-overlay band coverage.
|
|
1345
|
+
bins : int, optional
|
|
1346
|
+
Discretize a numeric x into ``bins`` equal-width intervals before
|
|
1347
|
+
plotting, so the discrete-style summary (geom_jitter + dispersion
|
|
1348
|
+
marker) applies. Mutually exclusive with ``breaks`` (which wins).
|
|
1349
|
+
No-op when x is already discrete or non-numeric.
|
|
1350
|
+
labels : list of str, optional
|
|
1351
|
+
Custom labels for the discrete x levels produced by ``bins`` /
|
|
1352
|
+
``breaks``. Length must equal ``bins`` (when given with bins) or
|
|
1353
|
+
``len(breaks) - 1`` (when given with breaks).
|
|
1354
|
+
breaks : list of float, optional
|
|
1355
|
+
Explicit cut points for discretizing numeric x. Takes precedence
|
|
1356
|
+
over ``bins`` when both are given (a ``UserWarning`` is emitted).
|
|
1357
|
+
spread : {None, "ci", "sterr", "stdev", "range", "iqr", "quartiles", "no"}, default None
|
|
1358
|
+
Dispersion marker drawn in the discrete-x branch alongside
|
|
1359
|
+
``geom_jitter``. Mirrors R-flexplot's ``spread``.
|
|
1360
|
+
- ``None`` / ``"quartiles"`` / ``"iqr"``: median ± Q1/Q3 IQR
|
|
1361
|
+
(R's default for discrete x).
|
|
1362
|
+
- ``"ci"``: bootstrap CI on the mean.
|
|
1363
|
+
- ``"sterr"``: mean ± 1.96 × standard error of the mean.
|
|
1364
|
+
- ``"stdev"``: mean ± 1 SD as a pointrange.
|
|
1365
|
+
- ``"range"``: min-max range.
|
|
1366
|
+
- ``"no"``: no summary layer at all.
|
|
1367
|
+
sample : int, optional
|
|
1368
|
+
Subsample N rows for the plotnine layers (scatter / jitter) while
|
|
1369
|
+
keeping the smoother fits on the full DataFrame. No-op when
|
|
1370
|
+
``N >= len(data)``. Deterministic via ``np.random.default_rng(0)``.
|
|
1371
|
+
ghost_line : {"red", "dashed", "slope1", None}, default None
|
|
1372
|
+
Reference line drawn after the main layers. ``"red"`` for a solid
|
|
1373
|
+
red threshold at y=0; ``"dashed"`` for a black dashed reference
|
|
1374
|
+
at y=0; ``"slope1"`` for a diagonal slope=1 reference line for
|
|
1375
|
+
prediction-vs-observed overlays (v0.7.3+).
|
|
1376
|
+
plot_type : {"scatter", "line", "boxplot", "bar", "histogram", "qq", "density", "violin", None}, default None
|
|
1377
|
+
Explicit geom override. Bypasses the auto-dispatch. For
|
|
1378
|
+
intercept-only formulas, ``"histogram"`` (default), ``"qq"``,
|
|
1379
|
+
``"density"``, ``"boxplot"``, and ``"violin"`` produce univariate
|
|
1380
|
+
distribution plots.
|
|
1381
|
+
return_data : bool, default False
|
|
1382
|
+
When ``True``, return ``{"plot": ggplot, "data": DataFrame}``
|
|
1383
|
+
instead of just the plot. Useful with ``sample=`` to know which
|
|
1384
|
+
rows were plotted.
|
|
1385
|
+
ghost_reference : pd.DataFrame, optional
|
|
1386
|
+
Reference dataset to overlay on the same axes. Two patterns:
|
|
1387
|
+
- Columns ``(x, y)``: draws a gray geom_point layer (reference scatter).
|
|
1388
|
+
- Columns ``(x, "pred")``: draws a red dashed geom_line (prediction line).
|
|
1389
|
+
plot_string : dict, optional
|
|
1390
|
+
Override the axis/legend labels derived from the formula. Accepts
|
|
1391
|
+
keys ``x``, ``y``, ``title``, ``subtitle``, ``caption``, ``color``.
|
|
1392
|
+
related : bool, default False
|
|
1393
|
+
R-flexplot's paired-samples flag. When True and the formula is
|
|
1394
|
+
``y ~ x`` with a two-level categorical predictor, the plot shows
|
|
1395
|
+
paired difference scores (level2 - level1) against x = 1, with a
|
|
1396
|
+
reference line at 0 and a dispersion marker. Requires equal group
|
|
1397
|
+
sizes and no color or panel variables. Raises ``ValueError`` when
|
|
1398
|
+
the precondition is not met.
|
|
1399
|
+
interaction_model : bool, default False
|
|
1400
|
+
When ``True`` and the formula contains ``*`` or ``:`` syntax, fit
|
|
1401
|
+
a statsmodels OLS with the actual interaction term and overlay
|
|
1402
|
+
non-parallel per-color-group regression lines (rather than the
|
|
1403
|
+
default additive fit with parallel slopes). Suppresses the
|
|
1404
|
+
"additive fit" UserWarning when set. Falls back to the additive
|
|
1405
|
+
path when the formula has no interaction term, no separate color
|
|
1406
|
+
group, or only one color level.
|
|
1407
|
+
**kwargs
|
|
1408
|
+
Reserved for future extension.
|
|
1409
|
+
|
|
1410
|
+
Returns
|
|
1411
|
+
-------
|
|
1412
|
+
plotnine.ggplot
|
|
1413
|
+
The composed plot object. Call ``.draw()`` to render or ``.save()``
|
|
1414
|
+
to write to disk.
|
|
1415
|
+
|
|
1416
|
+
Notes
|
|
1417
|
+
-----
|
|
1418
|
+
For the numeric-vs-binary branch (binomial GLM), the band is always drawn
|
|
1419
|
+
on the response (probability) scale; plotnine handles the inverse-link
|
|
1420
|
+
transformation internally. Numeric binary ``[0, 1]`` y (v0.6.1+) and
|
|
1421
|
+
string binary y both route to the binomial branch. Explicit
|
|
1422
|
+
``method="logistic"`` bypasses the binary pre-check and forces the
|
|
1423
|
+
numeric branch with a parametric logistic GLM.
|
|
1424
|
+
|
|
1425
|
+
Interaction syntax (``*``, ``:``) is parsed since v0.6.2. The default
|
|
1426
|
+
fit remains **additive** (parallel slopes per color group); a
|
|
1427
|
+
``UserWarning`` is emitted whenever interaction syntax is detected.
|
|
1428
|
+
Pass ``interaction_model=True`` (v0.7.0+) to fit the actual
|
|
1429
|
+
interaction term and overlay non-parallel per-color-group regression
|
|
1430
|
+
lines; this also suppresses the additive-fit warning.
|
|
1431
|
+
|
|
1432
|
+
Examples
|
|
1433
|
+
--------
|
|
1434
|
+
>>> import pandas as pd
|
|
1435
|
+
>>> import numpy as np
|
|
1436
|
+
>>> from pyflexplot import flexplot
|
|
1437
|
+
>>> rng = np.random.default_rng(0)
|
|
1438
|
+
>>> df = pd.DataFrame({"x": rng.normal(size=100), "y": rng.normal(size=100)})
|
|
1439
|
+
>>> p = flexplot("y ~ x", data=df)
|
|
1440
|
+
>>> isinstance(p, ggplot)
|
|
1441
|
+
True
|
|
1442
|
+
|
|
1443
|
+
With uncertainty bands:
|
|
1444
|
+
|
|
1445
|
+
>>> p = flexplot("y ~ x", data=df, bands=[0.5, 0.8, 0.95])
|
|
1446
|
+
>>> any(isinstance(layer.geom, geom_smooth) for layer in p.layers)
|
|
1447
|
+
True
|
|
1448
|
+
|
|
1449
|
+
With overlay smoothers:
|
|
1450
|
+
|
|
1451
|
+
>>> p = flexplot(
|
|
1452
|
+
... "y ~ x", data=df,
|
|
1453
|
+
... overlay=[{"method": "loess", "label": "LOESS smoother"}],
|
|
1454
|
+
... )
|
|
1455
|
+
>>> smooth_layers = [l for l in p.layers if isinstance(l.geom, geom_smooth)]
|
|
1456
|
+
>>> len(smooth_layers) >= 2
|
|
1457
|
+
True
|
|
1458
|
+
|
|
1459
|
+
Auto-binning numeric x (v0.6.4):
|
|
1460
|
+
|
|
1461
|
+
>>> df2 = pd.DataFrame({
|
|
1462
|
+
... "x": rng.uniform(0, 100, size=80),
|
|
1463
|
+
... "y": rng.normal(size=80),
|
|
1464
|
+
... })
|
|
1465
|
+
>>> p = flexplot("y ~ x", data=df2, bins=4)
|
|
1466
|
+
>>> any(isinstance(layer.geom, geom_jitter) for layer in p.layers)
|
|
1467
|
+
True
|
|
1468
|
+
|
|
1469
|
+
Polynomial fit on a non-linear signal (v0.6.4):
|
|
1470
|
+
|
|
1471
|
+
>>> df3 = pd.DataFrame({
|
|
1472
|
+
... "x": np.linspace(-3, 3, 60),
|
|
1473
|
+
... "y": np.linspace(-3, 3, 60) ** 2 + rng.normal(scale=0.3, size=60),
|
|
1474
|
+
... })
|
|
1475
|
+
>>> p = flexplot("y ~ x", data=df3, method="polynomial")
|
|
1476
|
+
>>> any(isinstance(layer.geom, geom_line) for layer in p.layers)
|
|
1477
|
+
True
|
|
1478
|
+
"""
|
|
1479
|
+
if method not in _VALID_FLEXPLOT_METHODS:
|
|
1480
|
+
raise ValueError(
|
|
1481
|
+
f"method must be one of {sorted(_VALID_FLEXPLOT_METHODS)}; got {method!r}. "
|
|
1482
|
+
"Pass 'auto' for the default behaviour (LM for numeric-vs-numeric, "
|
|
1483
|
+
"binomial GLM for numeric-vs-binary)."
|
|
1484
|
+
)
|
|
1485
|
+
validate_uncertainty_params(uncertainty, level, bands, method)
|
|
1486
|
+
overlay_specs = _normalize_overlay(overlay)
|
|
1487
|
+
if spread is not None and spread not in (s for s in _VALID_SPREAD if s):
|
|
1488
|
+
raise ValueError(
|
|
1489
|
+
f"spread must be one of {sorted(s for s in _VALID_SPREAD if s)} "
|
|
1490
|
+
f"or None; got {spread!r}."
|
|
1491
|
+
)
|
|
1492
|
+
|
|
1493
|
+
variables = parse_flexplot_formula(formula)
|
|
1494
|
+
if variables.get("has_interaction") and not interaction_model:
|
|
1495
|
+
# v0.6.x default: parser accepts interaction syntax (R-compatible)
|
|
1496
|
+
# but the fit remains additive (parallel slopes per color group).
|
|
1497
|
+
# Warn so users aren't misled. When interaction_model=True is
|
|
1498
|
+
# explicit, the fit uses the actual interaction term via
|
|
1499
|
+
# _add_interaction_smooth() and no warning is needed (v0.7.0+).
|
|
1500
|
+
warnings.warn(
|
|
1501
|
+
f"Interaction syntax detected in formula {formula!r} but flexplot's "
|
|
1502
|
+
f"default fit is additive (parallel slopes per color group). "
|
|
1503
|
+
f"Pass `interaction_model=True` for true non-parallel slopes. "
|
|
1504
|
+
f"To suppress this warning, write the formula without `*` or `:`.",
|
|
1505
|
+
UserWarning,
|
|
1506
|
+
stacklevel=2,
|
|
1507
|
+
)
|
|
1508
|
+
|
|
1509
|
+
# --- Formula-function evaluation (R ``formula_functions`` parity) ---
|
|
1510
|
+
# Detect terms like ``log(x)``, ``sqrt(x)``, ``poly(x, 2)``, ``I(x**2)``
|
|
1511
|
+
# in the right-hand side, apply the whitelisted function to ``data``,
|
|
1512
|
+
# store the result in a column named after the inner variable, and
|
|
1513
|
+
# rewrite the formula so downstream code sees the simpler name.
|
|
1514
|
+
transformed_data, transformed_formula, transformed_terms = (
|
|
1515
|
+
_apply_formula_functions(formula, data)
|
|
1516
|
+
)
|
|
1517
|
+
if transformed_terms:
|
|
1518
|
+
# Re-parse the rewritten formula; variables dict now reflects the
|
|
1519
|
+
# simpler (un-transed) term names.
|
|
1520
|
+
variables = parse_flexplot_formula(transformed_formula)
|
|
1521
|
+
# Preserve the interaction flag from the original parse (it was
|
|
1522
|
+
# already detected before the function-rewrite pass).
|
|
1523
|
+
if "has_interaction" not in variables:
|
|
1524
|
+
variables["has_interaction"] = False
|
|
1525
|
+
formula = transformed_formula
|
|
1526
|
+
data = transformed_data
|
|
1527
|
+
|
|
1528
|
+
_validate_data_for_plot(
|
|
1529
|
+
formula, data, variables, intercept_only=variables.get("intercept_only", False)
|
|
1530
|
+
)
|
|
1531
|
+
|
|
1532
|
+
y = variables["y"]
|
|
1533
|
+
x = variables["x"]
|
|
1534
|
+
color = variables["color"]
|
|
1535
|
+
given = variables["given"]
|
|
1536
|
+
intercept_only = variables.get("intercept_only", False)
|
|
1537
|
+
|
|
1538
|
+
# --- Auto-categorize low-cardinality numeric predictors ---
|
|
1539
|
+
# R-flexplot's ``convert_if_less_than_five`` turns numeric variables
|
|
1540
|
+
# with <5 unique values into ordered factors. This must run BEFORE
|
|
1541
|
+
# the type-detection / binning below so the low-cardinality numeric
|
|
1542
|
+
# path is taken (discrete x branch) rather than the high-cardinality
|
|
1543
|
+
# numeric path (LM / loess smoother).
|
|
1544
|
+
if not intercept_only and x is not None:
|
|
1545
|
+
cat_targets = [v for v in [x, color, *given] if v is not None]
|
|
1546
|
+
# Avoid converting columns that already are transformed to a
|
|
1547
|
+
# ``_binned`` string — those are intentionally stringified above.
|
|
1548
|
+
cat_targets = [v for v in cat_targets if not v.endswith("_binned")]
|
|
1549
|
+
if cat_targets:
|
|
1550
|
+
data = _convert_low_cardinality_to_categorical(data, cat_targets)
|
|
1551
|
+
|
|
1552
|
+
# --- Optional subsampling (v0.6.5+) ---
|
|
1553
|
+
# When ``sample=N`` is set and N < len(data), subsample to N rows for
|
|
1554
|
+
# plotting only. Subsequent smoother fits still use the FULL data so the
|
|
1555
|
+
# fit isn't degraded by the subsample. We track ``_sampled_df`` for
|
|
1556
|
+
# return_data= so the caller knows which rows were plotted.
|
|
1557
|
+
if sample is not None:
|
|
1558
|
+
if not isinstance(sample, int) or isinstance(sample, bool):
|
|
1559
|
+
raise TypeError(
|
|
1560
|
+
f"sample must be an int >= 1; got {type(sample).__name__} ({sample!r})."
|
|
1561
|
+
)
|
|
1562
|
+
if sample < 1:
|
|
1563
|
+
raise ValueError(f"sample must be >= 1; got {sample}.")
|
|
1564
|
+
if sample is not None and sample < len(data):
|
|
1565
|
+
rng_sample = np.random.default_rng(0) # deterministic for reproducibility
|
|
1566
|
+
sampled_idx = rng_sample.choice(len(data), size=sample, replace=False)
|
|
1567
|
+
sampled_idx = np.sort(sampled_idx)
|
|
1568
|
+
plot_input_df = data.iloc[sampled_idx].reset_index(drop=True)
|
|
1569
|
+
fit_input_df = data # full data; smoother fits unchanged
|
|
1570
|
+
else:
|
|
1571
|
+
plot_input_df = data
|
|
1572
|
+
fit_input_df = data
|
|
1573
|
+
|
|
1574
|
+
# --- Multivariate slotting / auto-binning for slot 2+ numeric predictors ---
|
|
1575
|
+
# Mirrors R's ``flexplot_break_me``: a numeric color or given variable
|
|
1576
|
+
# with more than ``bins`` unique values is binned into ``<name>_binned``
|
|
1577
|
+
# and that column drives the color aesthetic / facets. Re-raises on
|
|
1578
|
+
# formulas with 3+ non-given predictors.
|
|
1579
|
+
if not intercept_only:
|
|
1580
|
+
slot_data, binned_color, binned_given, _bin_count = (
|
|
1581
|
+
_slot_bin_numeric_predictors(
|
|
1582
|
+
fit_input_df, formula, bins=bins, breaks=breaks, labels=labels
|
|
1583
|
+
)
|
|
1584
|
+
)
|
|
1585
|
+
# Apply the binning to both plot and fit inputs.
|
|
1586
|
+
binned_cols = [
|
|
1587
|
+
c for c in slot_data.columns
|
|
1588
|
+
if c not in fit_input_df.columns and c.endswith("_binned")
|
|
1589
|
+
]
|
|
1590
|
+
# The helper returns ``slot_data`` containing the original columns
|
|
1591
|
+
# PLUS any ``<name>_binned`` ones. We merge them back into both
|
|
1592
|
+
# ``plot_input_df`` and ``fit_input_df`` so the aesthetic mappings
|
|
1593
|
+
# and the smoother fits see the binned columns.
|
|
1594
|
+
if binned_cols:
|
|
1595
|
+
for col in binned_cols:
|
|
1596
|
+
# Original source column for the binned copy.
|
|
1597
|
+
src = col[: -len("_binned")]
|
|
1598
|
+
if src in fit_input_df.columns:
|
|
1599
|
+
# Carry over the binned values from slot_data; they are
|
|
1600
|
+
# a deterministic function of the original numeric
|
|
1601
|
+
# column, so the alignment is index-based.
|
|
1602
|
+
plot_input_df[col] = slot_data[col].to_numpy()
|
|
1603
|
+
fit_input_df[col] = slot_data[col].to_numpy()
|
|
1604
|
+
# Update variables' color / given to point at the binned cols
|
|
1605
|
+
# so the aes / facet wiring below uses them.
|
|
1606
|
+
if binned_color is not None and color is not None:
|
|
1607
|
+
variables["color"] = binned_color
|
|
1608
|
+
color = binned_color
|
|
1609
|
+
if binned_given:
|
|
1610
|
+
# Replace ``given`` with the binned variants while keeping
|
|
1611
|
+
# the same length / ordering as the original list.
|
|
1612
|
+
new_given: List[Optional[str]] = []
|
|
1613
|
+
for orig in variables["given"]:
|
|
1614
|
+
if orig is None:
|
|
1615
|
+
new_given.append(None)
|
|
1616
|
+
continue
|
|
1617
|
+
candidate = f"{orig}_binned"
|
|
1618
|
+
if candidate in fit_input_df.columns:
|
|
1619
|
+
new_given.append(candidate)
|
|
1620
|
+
else:
|
|
1621
|
+
new_given.append(orig)
|
|
1622
|
+
variables["given"] = new_given
|
|
1623
|
+
given = new_given
|
|
1624
|
+
|
|
1625
|
+
if intercept_only:
|
|
1626
|
+
# Intercept-only: show a univariate distribution of y.
|
|
1627
|
+
# R-flexplot supports histogram/qq/density/boxplot/violin via plot.type.
|
|
1628
|
+
p = _plot_univariate(plot_input_df, y, plot_type=plot_type, bins=bins)
|
|
1629
|
+
if return_data:
|
|
1630
|
+
return {"plot": p, "data": plot_input_df}
|
|
1631
|
+
return p
|
|
1632
|
+
|
|
1633
|
+
if not isinstance(related, bool):
|
|
1634
|
+
raise TypeError(f"related must be a bool; got {type(related).__name__}.")
|
|
1635
|
+
|
|
1636
|
+
# --- Related-samples / paired difference plot (R-flexplot related=T) ---
|
|
1637
|
+
# Only valid for y ~ x where x is a two-level grouping variable and there
|
|
1638
|
+
# are no color/given facets. We replace the data with paired difference
|
|
1639
|
+
# scores (level2 - level1) and draw a univariate difference plot.
|
|
1640
|
+
if related:
|
|
1641
|
+
if color is not None or len(given) > 0:
|
|
1642
|
+
raise ValueError(
|
|
1643
|
+
"related=True is only supported for formulas with a single "
|
|
1644
|
+
"predictor and no color or panel variables (e.g., 'y ~ x')."
|
|
1645
|
+
)
|
|
1646
|
+
if x is None:
|
|
1647
|
+
raise ValueError("related=True requires a predictor variable.")
|
|
1648
|
+
|
|
1649
|
+
x_series = plot_input_df[x]
|
|
1650
|
+
# Ensure a categorical-style grouping variable with exactly two levels.
|
|
1651
|
+
if pd.api.types.is_numeric_dtype(x_series) and x_series.nunique(dropna=True) == 2:
|
|
1652
|
+
x_series = x_series.astype(str)
|
|
1653
|
+
levs = sorted(x_series.dropna().unique())
|
|
1654
|
+
if len(levs) != 2:
|
|
1655
|
+
raise ValueError(
|
|
1656
|
+
f"related=True requires exactly 2 levels of the predictor; "
|
|
1657
|
+
f"{x!r} has {len(levs)} levels."
|
|
1658
|
+
)
|
|
1659
|
+
|
|
1660
|
+
groups = {
|
|
1661
|
+
lev: plot_input_df.loc[x_series == lev, y].reset_index(drop=True)
|
|
1662
|
+
for lev in levs
|
|
1663
|
+
}
|
|
1664
|
+
sizes = [len(g) for g in groups.values()]
|
|
1665
|
+
if len(set(sizes)) != 1:
|
|
1666
|
+
raise ValueError(
|
|
1667
|
+
"related=True requires equal group sizes to compute paired "
|
|
1668
|
+
f"differences; got sizes {dict(zip(levs, sizes))}."
|
|
1669
|
+
)
|
|
1670
|
+
|
|
1671
|
+
diff_label = f"Difference ({levs[1]}-{levs[0]})"
|
|
1672
|
+
related_df = pd.DataFrame({diff_label: groups[levs[1]].to_numpy() - groups[levs[0]].to_numpy()})
|
|
1673
|
+
alpha_rel = alpha if alpha is not None else 0.5
|
|
1674
|
+
p = _plot_related(
|
|
1675
|
+
related_df,
|
|
1676
|
+
diff_label,
|
|
1677
|
+
spread,
|
|
1678
|
+
plot_type,
|
|
1679
|
+
jitter,
|
|
1680
|
+
alpha_rel,
|
|
1681
|
+
raw_data,
|
|
1682
|
+
)
|
|
1683
|
+
if return_data:
|
|
1684
|
+
return {"plot": p, "data": related_df}
|
|
1685
|
+
return p
|
|
1686
|
+
|
|
1687
|
+
# Reject 3+ given variables: the formula parser accepts them but only
|
|
1688
|
+
# two are actually used (facet_grid takes at most two). Better to fail
|
|
1689
|
+
# loudly than silently drop the third.
|
|
1690
|
+
if len(given) > 2:
|
|
1691
|
+
raise ValueError(
|
|
1692
|
+
f"Formula {formula!r} has {len(given)} given variables after '|': "
|
|
1693
|
+
f"{given}. flexplot supports at most 2 given variables "
|
|
1694
|
+
"(a row facet and a column facet)."
|
|
1695
|
+
)
|
|
1696
|
+
|
|
1697
|
+
# Determine variable types
|
|
1698
|
+
is_y_numeric = pd.api.types.is_numeric_dtype(data[y])
|
|
1699
|
+
|
|
1700
|
+
# Numeric-x binning: if bins / breaks / labels are provided and x is
|
|
1701
|
+
# numeric (and not already auto-discrete), discretize x so the
|
|
1702
|
+
# discrete-style summary applies. Validation:
|
|
1703
|
+
# - bins: int >= 2 (>= 2 needed to be meaningful).
|
|
1704
|
+
# - breaks: list of floats, len >= 2, monotonically increasing, span the
|
|
1705
|
+
# x range.
|
|
1706
|
+
# - labels: list of strings, len = len(breaks) - 1 when provided with
|
|
1707
|
+
# breaks, or len = bins when provided with bins.
|
|
1708
|
+
# Mutual precedence: breaks > bins (breaks overrides bins when both set).
|
|
1709
|
+
# Validation lives in _validate_binning_params.
|
|
1710
|
+
if not _is_discrete(plot_input_df[x]) and pd.api.types.is_numeric_dtype(plot_input_df[x]):
|
|
1711
|
+
_validate_binning_params(bins, labels, breaks, plot_input_df[x])
|
|
1712
|
+
plot_df, x_discretized = _maybe_bin_numeric_x(
|
|
1713
|
+
plot_input_df, x, bins=bins, labels=labels, breaks=breaks
|
|
1714
|
+
)
|
|
1715
|
+
if x_discretized:
|
|
1716
|
+
is_x_discrete = True
|
|
1717
|
+
else:
|
|
1718
|
+
plot_df = plot_input_df.copy()
|
|
1719
|
+
is_x_discrete = False
|
|
1720
|
+
else:
|
|
1721
|
+
plot_df = plot_input_df.copy()
|
|
1722
|
+
is_x_discrete = _is_discrete(plot_input_df[x])
|
|
1723
|
+
# Convert numeric discrete X to string/categorical so plotnine treats x-axis as discrete levels
|
|
1724
|
+
if is_x_discrete and pd.api.types.is_numeric_dtype(plot_df[x]):
|
|
1725
|
+
plot_df[x] = plot_df[x].astype(str)
|
|
1726
|
+
|
|
1727
|
+
# Binary-0/1 pre-check: a numeric y whose unique values are a subset of
|
|
1728
|
+
# {0, 1} should be treated as a binary outcome for binomial smoothing,
|
|
1729
|
+
# regardless of pandas' is_numeric_dtype (which returns True for int).
|
|
1730
|
+
# Without this pre-check, numeric binary y would fall into the LM/loess
|
|
1731
|
+
# branch and the binomial GLM branch would only fire for non-numeric y
|
|
1732
|
+
# (where the .astype(float) below would raise first).
|
|
1733
|
+
# BUT: explicit method='logistic' bypasses this and routes through the
|
|
1734
|
+
# numeric branch with a logistic GLM (see _add_parametric_smooth).
|
|
1735
|
+
y_is_binary = False
|
|
1736
|
+
if is_y_numeric and method not in {"logistic", "glmer"}:
|
|
1737
|
+
try:
|
|
1738
|
+
unique_y = pd.Series(plot_input_df[y].dropna().astype(float)).unique()
|
|
1739
|
+
except (ValueError, TypeError):
|
|
1740
|
+
unique_y = None
|
|
1741
|
+
y_is_binary = (
|
|
1742
|
+
unique_y is not None
|
|
1743
|
+
and len(unique_y) == 2
|
|
1744
|
+
and set(unique_y).issubset({0.0, 1.0})
|
|
1745
|
+
)
|
|
1746
|
+
|
|
1747
|
+
# Build the base aesthetic with color/group when needed so all geoms pick it up.
|
|
1748
|
+
aes_kwargs = {"x": x, "y": y}
|
|
1749
|
+
if color:
|
|
1750
|
+
# Convert numeric discrete color variable (like pclass = 1, 2, 3) to string/categorical
|
|
1751
|
+
# so plotnine uses a discrete color palette rather than a continuous gradient.
|
|
1752
|
+
if pd.api.types.is_numeric_dtype(plot_df[color]) and _is_discrete(plot_df[color]):
|
|
1753
|
+
plot_df[color] = plot_df[color].astype(str)
|
|
1754
|
+
aes_kwargs["color"] = color
|
|
1755
|
+
aes_kwargs["group"] = color
|
|
1756
|
+
p = ggplot(plot_df, aes(**aes_kwargs))
|
|
1757
|
+
|
|
1758
|
+
# --- Optional plot_type override (v0.6.5+) ---
|
|
1759
|
+
# Bypasses the auto-dispatch and forces a specific geom. Useful when
|
|
1760
|
+
# the user knows they want a boxplot regardless of how x is shaped, or
|
|
1761
|
+
# when the auto-dispatch picks the wrong branch because the data
|
|
1762
|
+
# violates heuristics (e.g. 11 unique values in x rather than 10).
|
|
1763
|
+
if plot_type is not None:
|
|
1764
|
+
if plot_type not in _VALID_PLOT_TYPES:
|
|
1765
|
+
raise ValueError(
|
|
1766
|
+
f"plot_type must be one of {sorted(_VALID_PLOT_TYPES)}; got {plot_type!r}."
|
|
1767
|
+
)
|
|
1768
|
+
if plot_type == "scatter":
|
|
1769
|
+
p += geom_point(alpha=0.5)
|
|
1770
|
+
elif plot_type == "line":
|
|
1771
|
+
p += geom_line()
|
|
1772
|
+
elif plot_type == "boxplot":
|
|
1773
|
+
p += geom_boxplot()
|
|
1774
|
+
elif plot_type == "violin":
|
|
1775
|
+
p += geom_violin()
|
|
1776
|
+
elif plot_type == "bar":
|
|
1777
|
+
# plotnine's geom_bar doesn't accept fun=; use stat_summary
|
|
1778
|
+
# with fun_y=np.mean + geom="bar" to get a bar chart of
|
|
1779
|
+
# group means per x level.
|
|
1780
|
+
p += stat_summary(fun_y=np.mean, geom="bar")
|
|
1781
|
+
# Skip the auto-dispatch below.
|
|
1782
|
+
skip_dispatch = True
|
|
1783
|
+
else:
|
|
1784
|
+
skip_dispatch = False
|
|
1785
|
+
|
|
1786
|
+
# --- jitter / alpha / raw_data resolution (v0.8.0+, R-parity) ---
|
|
1787
|
+
# R ``match_jitter_categorical``: None + categorical x -> (0.2, 0);
|
|
1788
|
+
# None + numeric x -> (0, 0); True -> (0.2, 0); False -> (0, 0);
|
|
1789
|
+
# numeric length-1 -> (x, 0); length-2 -> (x, y). We delegate to the
|
|
1790
|
+
# helper for the categorical-numeric split. The check uses the
|
|
1791
|
+
# POST-binning ``is_x_discrete`` (which is True when ``bins=`` /
|
|
1792
|
+
# ``breaks=`` discretized x, or when low-cardinality conversion kicked
|
|
1793
|
+
# in) so a user who asks for `bins=4` always gets the categorical
|
|
1794
|
+
# jitter defaults regardless of the underlying numeric dtype.
|
|
1795
|
+
is_x_discrete_for_jitter = bool(is_x_discrete)
|
|
1796
|
+
if isinstance(jitter, (list, tuple)) and len(jitter) == 2:
|
|
1797
|
+
# Explicit numeric pair: bypass the R rule so users can still get
|
|
1798
|
+
# the exact jitter widths they want.
|
|
1799
|
+
jitter_xy = (float(jitter[0]), float(jitter[1]))
|
|
1800
|
+
elif isinstance(jitter, (int, float)) and not isinstance(jitter, bool):
|
|
1801
|
+
# Numeric length-1: pass through (R: ``c(.2)`` -> ``(0.2, 0)``).
|
|
1802
|
+
jitter_xy = (float(jitter), 0.0)
|
|
1803
|
+
else:
|
|
1804
|
+
jitter_xy = match_jitter_categorical(jitter, is_x_discrete_for_jitter)
|
|
1805
|
+
# alpha: explicit value (float in (0, 1]) wins everywhere; otherwise
|
|
1806
|
+
# use flexplot_alpha_default's categorical/numeric rule (0.2 for
|
|
1807
|
+
# categorical x, 0.5 for numeric x). Numeric binary y keeps the
|
|
1808
|
+
# legacy 0.3 default (parity with prior releases and a slightly
|
|
1809
|
+
# softer overlay on the tight 0/1 cluster).
|
|
1810
|
+
if alpha is not None:
|
|
1811
|
+
if not isinstance(alpha, (int, float)) or not (0 < alpha <= 1):
|
|
1812
|
+
raise ValueError(
|
|
1813
|
+
f"alpha must be a float in (0, 1]; got {alpha!r}."
|
|
1814
|
+
)
|
|
1815
|
+
alpha_point = float(alpha)
|
|
1816
|
+
else:
|
|
1817
|
+
if y_is_binary:
|
|
1818
|
+
alpha_point = 0.3
|
|
1819
|
+
else:
|
|
1820
|
+
alpha_point = flexplot_alpha_default(plot_input_df, x, y, alpha)
|
|
1821
|
+
|
|
1822
|
+
# Determine plot type.
|
|
1823
|
+
# Order matters:
|
|
1824
|
+
# 1. Binary 0/1 y must be detected before the generic numeric branch
|
|
1825
|
+
# (otherwise int/float [0, 1] y falls into LM/loess).
|
|
1826
|
+
# 2. Numeric X is "discrete" when _is_discrete() returns True (numeric
|
|
1827
|
+
# with <=10 unique values; post-05ac368 R-flexplot parity).
|
|
1828
|
+
if not skip_dispatch and y_is_binary and not is_x_discrete:
|
|
1829
|
+
# Binomial GLM branch — numeric binary outcome with numeric x.
|
|
1830
|
+
if raw_data:
|
|
1831
|
+
p += geom_point(alpha=alpha_point)
|
|
1832
|
+
p = _add_binomial_smooth(p, data, x, y, uncertainty, level, bands, color=color)
|
|
1833
|
+
if overlay_specs:
|
|
1834
|
+
p = _add_overlay_binomial(p, data, x, y, overlay_specs)
|
|
1835
|
+
|
|
1836
|
+
elif not skip_dispatch and not is_y_numeric and not is_x_discrete:
|
|
1837
|
+
# Non-numeric y (string/categorical) with numeric x. Validate as
|
|
1838
|
+
# numeric 0/1; reject anything that doesn't fit a {0, 1} subset.
|
|
1839
|
+
|
|
1840
|
+
try:
|
|
1841
|
+
unique_y = pd.Series(plot_df[y].dropna().astype(float)).unique()
|
|
1842
|
+
except (ValueError, TypeError):
|
|
1843
|
+
raise ValueError(
|
|
1844
|
+
f"Binomial smoothing requires a numeric binary 0/1 outcome; "
|
|
1845
|
+
f"{y!r} could not be converted to numeric"
|
|
1846
|
+
) from None
|
|
1847
|
+
if len(unique_y) != 2 or not set(unique_y).issubset({0.0, 1.0}):
|
|
1848
|
+
raise ValueError(
|
|
1849
|
+
f"Binomial smoothing requires a binary 0/1 outcome; {y!r} has "
|
|
1850
|
+
f"unique values: {sorted(unique_y)}"
|
|
1851
|
+
)
|
|
1852
|
+
if raw_data:
|
|
1853
|
+
p += geom_point(alpha=alpha_point)
|
|
1854
|
+
p = _add_binomial_smooth(p, data, x, y, uncertainty, level, bands, color=color)
|
|
1855
|
+
if overlay_specs:
|
|
1856
|
+
p = _add_overlay_binomial(p, data, x, y, overlay_specs)
|
|
1857
|
+
|
|
1858
|
+
elif not skip_dispatch and is_y_numeric and not is_x_discrete:
|
|
1859
|
+
if raw_data:
|
|
1860
|
+
p += geom_point(alpha=alpha_point)
|
|
1861
|
+
if interaction_model and variables.get("has_interaction") and color:
|
|
1862
|
+
# Non-parallel slopes per color group via statsmodels OLS with
|
|
1863
|
+
# the actual interaction term (e.g. y ~ x * color). v0.7.0+.
|
|
1864
|
+
p = _add_interaction_smooth(
|
|
1865
|
+
p, data, x, y, color, variables["all_x"],
|
|
1866
|
+
method, uncertainty, level, bands,
|
|
1867
|
+
)
|
|
1868
|
+
else:
|
|
1869
|
+
p = _add_numeric_smooth(
|
|
1870
|
+
p,
|
|
1871
|
+
data,
|
|
1872
|
+
x,
|
|
1873
|
+
y,
|
|
1874
|
+
method,
|
|
1875
|
+
uncertainty,
|
|
1876
|
+
level,
|
|
1877
|
+
bands,
|
|
1878
|
+
color=color,
|
|
1879
|
+
random_effects=random_effects,
|
|
1880
|
+
mixed_backend=mixed_backend,
|
|
1881
|
+
)
|
|
1882
|
+
if overlay_specs:
|
|
1883
|
+
p = _add_overlay_numeric(p, data, x, y, overlay_specs)
|
|
1884
|
+
|
|
1885
|
+
elif not skip_dispatch and is_y_numeric and is_x_discrete:
|
|
1886
|
+
if raw_data and jitter_xy[0] > 0:
|
|
1887
|
+
p += geom_jitter(width=jitter_xy[0], alpha=alpha_point)
|
|
1888
|
+
elif raw_data:
|
|
1889
|
+
p += geom_point(alpha=alpha_point)
|
|
1890
|
+
p = _add_discrete_summary(p, spread)
|
|
1891
|
+
|
|
1892
|
+
elif not skip_dispatch:
|
|
1893
|
+
if raw_data and jitter_xy != (0.0, 0.0):
|
|
1894
|
+
p += geom_jitter(width=jitter_xy[0], height=jitter_xy[1], alpha=alpha_point)
|
|
1895
|
+
elif raw_data:
|
|
1896
|
+
p += geom_point(alpha=alpha_point)
|
|
1897
|
+
|
|
1898
|
+
if len(given) == 1:
|
|
1899
|
+
p += facet_wrap(f"~{given[0]}")
|
|
1900
|
+
elif len(given) >= 2:
|
|
1901
|
+
p += facet_grid(f"{given[1]} ~ {given[0]}")
|
|
1902
|
+
|
|
1903
|
+
p += theme_bw()
|
|
1904
|
+
|
|
1905
|
+
# --- Optional plot.string override (v0.6.6+) ---
|
|
1906
|
+
# R-flexplot's plot.string is a dict of label overrides:
|
|
1907
|
+
# {"x": "Time (s)", "y": "Voltage (V)", "title": "Experiment 1"}
|
|
1908
|
+
# Validation: must be a dict with string keys and string values.
|
|
1909
|
+
if plot_string is not None:
|
|
1910
|
+
if not isinstance(plot_string, dict):
|
|
1911
|
+
raise TypeError(
|
|
1912
|
+
f"plot_string must be a dict of {{label: text}} overrides; "
|
|
1913
|
+
f"got {type(plot_string).__name__}."
|
|
1914
|
+
)
|
|
1915
|
+
bad = {k: type(v).__name__ for k, v in plot_string.items()
|
|
1916
|
+
if not isinstance(k, str) or not isinstance(v, str)}
|
|
1917
|
+
if bad:
|
|
1918
|
+
raise TypeError(
|
|
1919
|
+
f"plot_string keys and values must all be strings; "
|
|
1920
|
+
f"bad entries: {bad}"
|
|
1921
|
+
)
|
|
1922
|
+
# Apply via plotnine's labs(). Only known labels are passed through;
|
|
1923
|
+
# unknown keys are silently ignored (plotnine's labs() ignores
|
|
1924
|
+
# unknown keys but emits a warning we don't want to surface).
|
|
1925
|
+
labs_kwargs = {
|
|
1926
|
+
k: v for k, v in plot_string.items()
|
|
1927
|
+
if k in {"x", "y", "title", "subtitle", "caption", "color"}
|
|
1928
|
+
}
|
|
1929
|
+
if labs_kwargs:
|
|
1930
|
+
p += labs(**labs_kwargs)
|
|
1931
|
+
|
|
1932
|
+
# --- Optional ghost.line reference layer (v0.6.5+) ---
|
|
1933
|
+
# ghost_line="red": solid red reference line. Useful for highlighting a
|
|
1934
|
+
# threshold or a reference value (e.g. y=0, or y=mean(y)).
|
|
1935
|
+
# ghost_line="dashed": dashed black line. R's flexplot() uses this to
|
|
1936
|
+
# mark the slope=1 reference for prediction-vs-observed plots.
|
|
1937
|
+
# Both are drawn as geom_hline (horizontal), so they're 1D references
|
|
1938
|
+
# at y=0. For diagonal references (slope=1), future work.
|
|
1939
|
+
# --- ghost.line & ghost.reference (v0.8.0, R-parity) ---
|
|
1940
|
+
# R semantics: ghost.line is the COLOR of a line fit on a reference
|
|
1941
|
+
# panel that is repeated into every other panel (cross-panel
|
|
1942
|
+
# comparison). If ghost_reference is provided without an explicit
|
|
1943
|
+
# ghost_line, ghost_line defaults to "gray".
|
|
1944
|
+
has_grouping = len(given) >= 1 or color is not None
|
|
1945
|
+
|
|
1946
|
+
if ghost_reference is not None and not isinstance(ghost_reference, (dict, pd.DataFrame)):
|
|
1947
|
+
raise TypeError(
|
|
1948
|
+
f"ghost_reference must be None, a dict ({{given_var: level}}) for "
|
|
1949
|
+
f"panel reference selection, or a DataFrame for overlay; got "
|
|
1950
|
+
f"{type(ghost_reference).__name__}."
|
|
1951
|
+
)
|
|
1952
|
+
|
|
1953
|
+
if isinstance(ghost_reference, dict) and not has_grouping:
|
|
1954
|
+
raise TypeError(
|
|
1955
|
+
"ghost_reference dict requires a `| given` facet in the formula."
|
|
1956
|
+
)
|
|
1957
|
+
|
|
1958
|
+
if ghost_reference is not None and not isinstance(ghost_reference, pd.DataFrame) and ghost_line is None:
|
|
1959
|
+
ghost_line = "gray"
|
|
1960
|
+
if ghost_line is True:
|
|
1961
|
+
ghost_line = "gray"
|
|
1962
|
+
|
|
1963
|
+
if ghost_line is not None:
|
|
1964
|
+
if not isinstance(ghost_line, str):
|
|
1965
|
+
raise TypeError(
|
|
1966
|
+
f"ghost_line must be a color string, 'slope1', or None; "
|
|
1967
|
+
f"got {type(ghost_line).__name__}."
|
|
1968
|
+
)
|
|
1969
|
+
if has_grouping:
|
|
1970
|
+
# R-parity path: fit y ~ x on the reference subset and repeat
|
|
1971
|
+
# the predicted line into every panel/group, drawn in ghost_line's
|
|
1972
|
+
# color. Reference selection via ghost_reference dict
|
|
1973
|
+
# ({given_var: level}) or the first level of the first given
|
|
1974
|
+
# variable when absent.
|
|
1975
|
+
if x is None or is_x_discrete:
|
|
1976
|
+
# Ghost lines are only defined for numeric x fits.
|
|
1977
|
+
if not isinstance(ghost_line, str):
|
|
1978
|
+
raise TypeError("ghost_line must be a string.")
|
|
1979
|
+
ref_df: Optional[pd.DataFrame] = plot_input_df
|
|
1980
|
+
if ghost_reference is not None:
|
|
1981
|
+
if isinstance(ghost_reference, pd.DataFrame):
|
|
1982
|
+
# DataFrame overlay path (legacy) is handled AFTER this
|
|
1983
|
+
# block; dicts drive panel-referencing here.
|
|
1984
|
+
ref_df = None
|
|
1985
|
+
elif isinstance(ghost_reference, dict):
|
|
1986
|
+
for var, val in ghost_reference.items():
|
|
1987
|
+
if var not in plot_input_df.columns:
|
|
1988
|
+
raise ValueError(
|
|
1989
|
+
f"ghost_reference key {var!r} is not a "
|
|
1990
|
+
f"data column."
|
|
1991
|
+
)
|
|
1992
|
+
mask = (plot_input_df[var] == val) | (plot_input_df[var].astype(str) == str(val))
|
|
1993
|
+
if not mask.any():
|
|
1994
|
+
# Nearest-match fallback for numeric refs.
|
|
1995
|
+
if pd.api.types.is_numeric_dtype(plot_input_df[var]):
|
|
1996
|
+
idx = (plot_input_df[var] - val).abs().idxmin()
|
|
1997
|
+
mask = plot_input_df[var] == plot_input_df.loc[idx, var]
|
|
1998
|
+
else:
|
|
1999
|
+
mask = plot_input_df[var] == plot_input_df[var].iloc[0]
|
|
2000
|
+
ref_df = plot_input_df[mask]
|
|
2001
|
+
else:
|
|
2002
|
+
raise TypeError(
|
|
2003
|
+
"ghost_reference must be None, a dict "
|
|
2004
|
+
"({given_var: level}) for panel reference selection, "
|
|
2005
|
+
"or a DataFrame for overlay; got "
|
|
2006
|
+
f"{type(ghost_reference).__name__}."
|
|
2007
|
+
)
|
|
2008
|
+
if ref_df is not None and len(ref_df) > 1 and pd.api.types.is_numeric_dtype(ref_df[x]):
|
|
2009
|
+
gx = ref_df[x].to_numpy(dtype=float)
|
|
2010
|
+
gy = ref_df[y].to_numpy(dtype=float)
|
|
2011
|
+
valid = np.isfinite(gx) & np.isfinite(gy)
|
|
2012
|
+
gx_valid = gx[valid]
|
|
2013
|
+
gy_valid = gy[valid]
|
|
2014
|
+
if len(gx_valid) > 1:
|
|
2015
|
+
_X = np.column_stack([np.ones_like(gx_valid), gx_valid])
|
|
2016
|
+
_m = OLS(gy_valid, _X).fit()
|
|
2017
|
+
_x_eval = np.linspace(np.nanmin(gx_valid), np.nanmax(gx_valid), num=200)
|
|
2018
|
+
_y_eval = _m.predict(np.column_stack([np.ones_like(_x_eval), _x_eval]))
|
|
2019
|
+
ghost_df = pd.DataFrame({x: _x_eval, y: _y_eval})
|
|
2020
|
+
ghost_color = "black" if ghost_line == "dashed" else ghost_line
|
|
2021
|
+
p += geom_line(
|
|
2022
|
+
aes(x=x, y=y),
|
|
2023
|
+
data=ghost_df,
|
|
2024
|
+
color=ghost_color,
|
|
2025
|
+
linetype="dashed",
|
|
2026
|
+
inherit_aes=False,
|
|
2027
|
+
)
|
|
2028
|
+
else:
|
|
2029
|
+
# No grouping/facets: reference line at y=0 or diagonal slope1
|
|
2030
|
+
valid_unfaceted = {"red", "dashed", "slope1", "gray", "lightgray", "black", "blue", "darkgreen"}
|
|
2031
|
+
if ghost_line not in valid_unfaceted:
|
|
2032
|
+
raise ValueError(
|
|
2033
|
+
f"Without `| given` facets or color predictors, ghost_line must be 'gray', "
|
|
2034
|
+
f"'red', 'dashed', 'slope1', or None; got {ghost_line!r}. "
|
|
2035
|
+
f"Panel-repetition (R parity) requires a facet in the "
|
|
2036
|
+
f"formula."
|
|
2037
|
+
)
|
|
2038
|
+
if ghost_line == "slope1":
|
|
2039
|
+
from plotnine import geom_abline
|
|
2040
|
+
p += geom_abline(intercept=0, slope=1, color="black", linetype="dashed")
|
|
2041
|
+
else:
|
|
2042
|
+
ghost_color = "black" if ghost_line == "dashed" else ghost_line
|
|
2043
|
+
p += geom_hline(yintercept=0, color=ghost_color, linetype="dashed" if ghost_line == "dashed" else "solid")
|
|
2044
|
+
|
|
2045
|
+
# --- Optional ghost.reference overlay (v0.6.6+) ---
|
|
2046
|
+
# R-flexplot accepts ghost.reference as a DataFrame to overlay on the
|
|
2047
|
+
# same axes. Two common patterns:
|
|
2048
|
+
# 1. Reference scatter: columns matching x/y → draw geom_point in
|
|
2049
|
+
# light gray.
|
|
2050
|
+
# 2. Reference prediction line: columns (x, "pred") → draw geom_line
|
|
2051
|
+
# in a contrasting color.
|
|
2052
|
+
# We detect the pattern by checking if the DataFrame has columns
|
|
2053
|
+
# [x, y] or [x, "pred"].
|
|
2054
|
+
if ghost_reference is not None:
|
|
2055
|
+
# Dict form ({given_var: level}) was consumed by the ghost.line
|
|
2056
|
+
# panel-reference path above; only DataFrames reach this legacy
|
|
2057
|
+
# overlay path.
|
|
2058
|
+
if isinstance(ghost_reference, dict):
|
|
2059
|
+
# Dict = panel-reference selector, consumed by the ghost.line
|
|
2060
|
+
# block above. Here we only validate the facet requirement.
|
|
2061
|
+
if not has_grouping:
|
|
2062
|
+
raise TypeError(
|
|
2063
|
+
"ghost_reference dict requires a `| given` facet in the "
|
|
2064
|
+
"formula (it selects the reference panel)."
|
|
2065
|
+
)
|
|
2066
|
+
elif isinstance(ghost_reference, pd.DataFrame):
|
|
2067
|
+
pass # DataFrame overlay handled below.
|
|
2068
|
+
else:
|
|
2069
|
+
raise TypeError(
|
|
2070
|
+
f"ghost_reference must be None, a dict (panel reference), "
|
|
2071
|
+
f"or a pandas DataFrame (overlay); got "
|
|
2072
|
+
f"{type(ghost_reference).__name__}."
|
|
2073
|
+
)
|
|
2074
|
+
if isinstance(ghost_reference, pd.DataFrame) and x not in ghost_reference.columns:
|
|
2075
|
+
raise ValueError(
|
|
2076
|
+
f"ghost_reference DataFrame must have column {x!r} "
|
|
2077
|
+
f"(matching x in the formula); got columns "
|
|
2078
|
+
f"{list(ghost_reference.columns)}."
|
|
2079
|
+
)
|
|
2080
|
+
if isinstance(ghost_reference, dict):
|
|
2081
|
+
pass # dict refs don't carry overlay columns
|
|
2082
|
+
elif "pred" in ghost_reference.columns:
|
|
2083
|
+
# Prediction-line pattern: geom_line in red.
|
|
2084
|
+
p += geom_line(
|
|
2085
|
+
aes(x=x, y="pred"),
|
|
2086
|
+
data=ghost_reference,
|
|
2087
|
+
color="red",
|
|
2088
|
+
linetype="dashed",
|
|
2089
|
+
inherit_aes=False,
|
|
2090
|
+
)
|
|
2091
|
+
elif y in ghost_reference.columns:
|
|
2092
|
+
# Reference-scatter pattern: geom_point in light gray.
|
|
2093
|
+
p += geom_point(
|
|
2094
|
+
data=ghost_reference,
|
|
2095
|
+
color="gray",
|
|
2096
|
+
alpha=0.4,
|
|
2097
|
+
inherit_aes=False,
|
|
2098
|
+
)
|
|
2099
|
+
else:
|
|
2100
|
+
raise ValueError(
|
|
2101
|
+
f"ghost_reference must have either column {y!r} (scatter) "
|
|
2102
|
+
f"or 'pred' (line); got {list(ghost_reference.columns)}."
|
|
2103
|
+
)
|
|
2104
|
+
|
|
2105
|
+
if return_data:
|
|
2106
|
+
return {"plot": p, "data": plot_input_df}
|
|
2107
|
+
return p
|
|
2108
|
+
|
|
2109
|
+
|
|
2110
|
+
def _lowess_predict(x_eval: np.ndarray, y: np.ndarray, x: np.ndarray) -> np.ndarray:
|
|
2111
|
+
"""Wrapper around statsmodels ``lowess`` that takes (x_eval, y_sorted_by_x)."""
|
|
2112
|
+
order = np.argsort(x)
|
|
2113
|
+
x_sorted = np.asarray(x)[order]
|
|
2114
|
+
y_sorted = np.asarray(y)[order]
|
|
2115
|
+
smoothed = lowess(y_sorted, x_sorted, return_sorted=False)
|
|
2116
|
+
return np.interp(x_eval, x_sorted, smoothed)
|
|
2117
|
+
|
|
2118
|
+
|
|
2119
|
+
def _add_numeric_smooth(
|
|
2120
|
+
p,
|
|
2121
|
+
data: pd.DataFrame,
|
|
2122
|
+
x: str,
|
|
2123
|
+
y: str,
|
|
2124
|
+
method: str,
|
|
2125
|
+
uncertainty: Optional[str],
|
|
2126
|
+
level: float,
|
|
2127
|
+
bands: Optional[List[float]],
|
|
2128
|
+
color: Optional[str] = None,
|
|
2129
|
+
random_effects: Optional[str] = None,
|
|
2130
|
+
mixed_backend: str = "auto",
|
|
2131
|
+
):
|
|
2132
|
+
"""Add fitted line + uncertainty band for numeric-vs-numeric.
|
|
2133
|
+
|
|
2134
|
+
Returns the plotnine plot object with the appropriate layers added.
|
|
2135
|
+
Caller is responsible for adding geom_point first.
|
|
2136
|
+
"""
|
|
2137
|
+
if uncertainty is None:
|
|
2138
|
+
# No fit at all — preserve the scatter only.
|
|
2139
|
+
return p
|
|
2140
|
+
|
|
2141
|
+
line_color = None if color else "blue"
|
|
2142
|
+
|
|
2143
|
+
# polynomial/quadratic/cubic are OLS fits with higher-order x terms;
|
|
2144
|
+
# logistic/poisson/Gamma are GLMs; rlm is robust regression. plotnine's
|
|
2145
|
+
# geom_smooth does NOT support all of these cleanly, so we route them
|
|
2146
|
+
# through statsmodels and add geom_line + geom_ribbon manually.
|
|
2147
|
+
if method in {
|
|
2148
|
+
"quadratic",
|
|
2149
|
+
"polynomial",
|
|
2150
|
+
"cubic",
|
|
2151
|
+
"logistic",
|
|
2152
|
+
"rlm",
|
|
2153
|
+
"poisson",
|
|
2154
|
+
"Gamma",
|
|
2155
|
+
"mixedlm",
|
|
2156
|
+
"lmer",
|
|
2157
|
+
"glmer",
|
|
2158
|
+
}:
|
|
2159
|
+
return _add_parametric_smooth(
|
|
2160
|
+
p,
|
|
2161
|
+
data,
|
|
2162
|
+
x,
|
|
2163
|
+
y,
|
|
2164
|
+
method,
|
|
2165
|
+
uncertainty,
|
|
2166
|
+
level,
|
|
2167
|
+
bands,
|
|
2168
|
+
random_effects=random_effects,
|
|
2169
|
+
mixed_backend=mixed_backend,
|
|
2170
|
+
)
|
|
2171
|
+
|
|
2172
|
+
use_loess = method == "loess"
|
|
2173
|
+
|
|
2174
|
+
# --- Nested bands (multiple ribbons via multiple geom_smooth layers) ---
|
|
2175
|
+
if bands is not None:
|
|
2176
|
+
levels = sorted(set(bands))
|
|
2177
|
+
for lvl in levels:
|
|
2178
|
+
kwargs = {"method": "loess" if use_loess else "lm", "level": lvl, "alpha": 0.15}
|
|
2179
|
+
if line_color:
|
|
2180
|
+
kwargs["color"] = line_color
|
|
2181
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2182
|
+
return p
|
|
2183
|
+
|
|
2184
|
+
# --- Single band ---
|
|
2185
|
+
if uncertainty == "ci":
|
|
2186
|
+
kwargs = {"method": "loess" if use_loess else "lm", "level": level}
|
|
2187
|
+
if line_color:
|
|
2188
|
+
kwargs["color"] = line_color
|
|
2189
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2190
|
+
return p
|
|
2191
|
+
|
|
2192
|
+
if uncertainty == "prediction":
|
|
2193
|
+
# Fit an OLS model, compute residual-based PI on a sorted-x grid,
|
|
2194
|
+
# and draw a ribbon + the fitted line.
|
|
2195
|
+
from scipy import stats as _scipy_stats
|
|
2196
|
+
|
|
2197
|
+
x_arr = data[x].to_numpy(dtype=float)
|
|
2198
|
+
y_arr = data[y].to_numpy(dtype=float)
|
|
2199
|
+
model = OLS(y_arr, sm.add_constant(x_arr)).fit()
|
|
2200
|
+
x_eval = np.sort(np.unique(x_arr))
|
|
2201
|
+
if x_eval.size < 2:
|
|
2202
|
+
# Degenerate: cannot draw a meaningful band.
|
|
2203
|
+
p += geom_line(aes(y=y), color="blue")
|
|
2204
|
+
return p
|
|
2205
|
+
yhat_eval = model.predict(sm.add_constant(x_eval))
|
|
2206
|
+
yhat_full = model.predict(sm.add_constant(x_arr))
|
|
2207
|
+
sigma = float(np.sqrt(np.mean((y_arr - yhat_full) ** 2)))
|
|
2208
|
+
z = float(_scipy_stats.norm.ppf(0.5 + level / 2))
|
|
2209
|
+
half_width = z * sigma
|
|
2210
|
+
ribbon_df = pd.DataFrame({
|
|
2211
|
+
x: x_eval,
|
|
2212
|
+
"__lower": yhat_eval - half_width,
|
|
2213
|
+
"__upper": yhat_eval + half_width,
|
|
2214
|
+
y: yhat_eval,
|
|
2215
|
+
})
|
|
2216
|
+
p += geom_ribbon(
|
|
2217
|
+
aes(ymin="__lower", ymax="__upper"),
|
|
2218
|
+
data=ribbon_df,
|
|
2219
|
+
alpha=0.2,
|
|
2220
|
+
fill="blue",
|
|
2221
|
+
inherit_aes=False,
|
|
2222
|
+
)
|
|
2223
|
+
p += geom_line(
|
|
2224
|
+
aes(y=y),
|
|
2225
|
+
data=ribbon_df,
|
|
2226
|
+
color="blue",
|
|
2227
|
+
inherit_aes=False,
|
|
2228
|
+
)
|
|
2229
|
+
return p
|
|
2230
|
+
|
|
2231
|
+
if uncertainty == "bootstrap":
|
|
2232
|
+
# Case-resampled bootstrap CI for the loess branch.
|
|
2233
|
+
x_arr = data[x].to_numpy(dtype=float)
|
|
2234
|
+
y_arr = data[y].to_numpy(dtype=float)
|
|
2235
|
+
x_eval, lower, upper = compute_bootstrap_ci(
|
|
2236
|
+
x_arr, y_arr,
|
|
2237
|
+
smooth_fn=lambda x_e, y_s: _lowess_predict(x_e, y_s, x_arr),
|
|
2238
|
+
n_resamples=200,
|
|
2239
|
+
level=level,
|
|
2240
|
+
random_state=None,
|
|
2241
|
+
)
|
|
2242
|
+
smoothed_line = _lowess_predict(x_eval, y_arr, x_arr)
|
|
2243
|
+
ribbon_df = pd.DataFrame({
|
|
2244
|
+
x: x_eval,
|
|
2245
|
+
"__lower": lower,
|
|
2246
|
+
"__upper": upper,
|
|
2247
|
+
y: smoothed_line,
|
|
2248
|
+
})
|
|
2249
|
+
p += geom_ribbon(
|
|
2250
|
+
aes(ymin="__lower", ymax="__upper"),
|
|
2251
|
+
data=ribbon_df,
|
|
2252
|
+
alpha=0.2,
|
|
2253
|
+
fill="blue",
|
|
2254
|
+
inherit_aes=False,
|
|
2255
|
+
)
|
|
2256
|
+
p += geom_line(
|
|
2257
|
+
aes(y=y),
|
|
2258
|
+
data=ribbon_df,
|
|
2259
|
+
color="blue",
|
|
2260
|
+
inherit_aes=False,
|
|
2261
|
+
)
|
|
2262
|
+
return p
|
|
2263
|
+
|
|
2264
|
+
# Should never reach here thanks to validate_uncertainty_params.
|
|
2265
|
+
return p
|
|
2266
|
+
|
|
2267
|
+
|
|
2268
|
+
def _add_parametric_smooth(
|
|
2269
|
+
p,
|
|
2270
|
+
data: pd.DataFrame,
|
|
2271
|
+
x: str,
|
|
2272
|
+
y: str,
|
|
2273
|
+
method: str,
|
|
2274
|
+
uncertainty: Optional[str],
|
|
2275
|
+
level: float,
|
|
2276
|
+
bands: Optional[List[float]],
|
|
2277
|
+
random_effects: Optional[str] = None,
|
|
2278
|
+
mixed_backend: str = "auto",
|
|
2279
|
+
):
|
|
2280
|
+
"""Add fitted line + CI ribbon for polynomial / cubic / logistic methods.
|
|
2281
|
+
|
|
2282
|
+
plotnine's ``geom_smooth(method="lm", ...)`` does NOT accept
|
|
2283
|
+
``formula=poly(x, k)`` cleanly, so we fit statsmodels directly and draw
|
|
2284
|
+
the line + ribbon manually. Mirrors the prediction/ bootstrap branches
|
|
2285
|
+
in ``_add_numeric_smooth``.
|
|
2286
|
+
|
|
2287
|
+
Methods:
|
|
2288
|
+
- "quadratic"/"polynomial": degree-2 OLS.
|
|
2289
|
+
- "cubic": degree-3 OLS.
|
|
2290
|
+
- "logistic": GLM with logit link on numeric binary y.
|
|
2291
|
+
- "mixedlm"/"lmer": linear mixed-effects with a random intercept.
|
|
2292
|
+
- "glmer": binomial mixed-effects with a random intercept.
|
|
2293
|
+
"""
|
|
2294
|
+
from scipy import stats as _scipy_stats
|
|
2295
|
+
|
|
2296
|
+
def _parse_random_effects_spec(spec: Optional[str], x_name: str):
|
|
2297
|
+
if not spec:
|
|
2298
|
+
raise ValueError(
|
|
2299
|
+
"Mixed-effects methods require random_effects=. "
|
|
2300
|
+
"Use a group column name (e.g., random_effects='school') "
|
|
2301
|
+
"or an lme4-style mini spec (e.g., '(1|school)' or "
|
|
2302
|
+
"'(1 + x|school)')."
|
|
2303
|
+
)
|
|
2304
|
+
text = str(spec).strip()
|
|
2305
|
+
if text in data.columns:
|
|
2306
|
+
return "1", text
|
|
2307
|
+
m = re.match(r"^\(\s*(.+?)\s*\|\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)$", text)
|
|
2308
|
+
if not m:
|
|
2309
|
+
raise ValueError(
|
|
2310
|
+
f"Invalid random_effects specification {spec!r}. "
|
|
2311
|
+
"Expected a column name or '(effects|group)'."
|
|
2312
|
+
)
|
|
2313
|
+
re_formula_raw, group_col = m.group(1).strip(), m.group(2).strip()
|
|
2314
|
+
if group_col not in data.columns:
|
|
2315
|
+
raise ValueError(
|
|
2316
|
+
f"random_effects group column {group_col!r} not found in data."
|
|
2317
|
+
)
|
|
2318
|
+
if re_formula_raw == "1":
|
|
2319
|
+
return "1", group_col
|
|
2320
|
+
re_formula = re_formula_raw.replace(x_name, "x")
|
|
2321
|
+
allowed = {"1 + x", "x", "0 + x", "1+x", "0+x"}
|
|
2322
|
+
if re_formula not in allowed:
|
|
2323
|
+
raise NotImplementedError(
|
|
2324
|
+
"Supported random_effects formulas are '(1|g)' and "
|
|
2325
|
+
"'(1 + x|g)' (or '(x|g)')."
|
|
2326
|
+
)
|
|
2327
|
+
if re_formula in {"x", "0 + x", "0+x"}:
|
|
2328
|
+
re_formula = "0 + x"
|
|
2329
|
+
else:
|
|
2330
|
+
re_formula = "1 + x"
|
|
2331
|
+
return re_formula, group_col
|
|
2332
|
+
|
|
2333
|
+
def _add_fe_band(ribbon_df: pd.DataFrame, fe_mean: np.ndarray, fe_cov: np.ndarray):
|
|
2334
|
+
if bands is not None:
|
|
2335
|
+
levels = sorted(set(bands))
|
|
2336
|
+
else:
|
|
2337
|
+
levels = [level]
|
|
2338
|
+
for lvl in levels:
|
|
2339
|
+
z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
|
|
2340
|
+
se = np.sqrt(np.maximum(np.einsum("ij,jk,ik->i", fe_mean, fe_cov, fe_mean), 0.0))
|
|
2341
|
+
ribbon_df[f"__lower_{lvl}"] = ribbon_df[y].to_numpy() - z * se
|
|
2342
|
+
ribbon_df[f"__upper_{lvl}"] = ribbon_df[y].to_numpy() + z * se
|
|
2343
|
+
for lvl in sorted(levels, reverse=True):
|
|
2344
|
+
alpha = 0.1 + 0.15 * (lvl / max(levels))
|
|
2345
|
+
p_local = geom_ribbon(
|
|
2346
|
+
aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
|
|
2347
|
+
data=ribbon_df,
|
|
2348
|
+
alpha=alpha,
|
|
2349
|
+
fill="blue",
|
|
2350
|
+
inherit_aes=False,
|
|
2351
|
+
)
|
|
2352
|
+
yield p_local
|
|
2353
|
+
|
|
2354
|
+
x_arr = data[x].to_numpy(dtype=float)
|
|
2355
|
+
y_arr = data[y].to_numpy(dtype=float)
|
|
2356
|
+
n = x_arr.size
|
|
2357
|
+
if n < 2:
|
|
2358
|
+
return p
|
|
2359
|
+
|
|
2360
|
+
if method in {"quadratic", "polynomial"}:
|
|
2361
|
+
# R-flexplot: both "polynomial" and "quadratic" are degree-2 OLS.
|
|
2362
|
+
X = np.column_stack([np.ones_like(x_arr), x_arr, x_arr ** 2])
|
|
2363
|
+
model = OLS(y_arr, X).fit()
|
|
2364
|
+
link_label = "polynomial (degree-2)"
|
|
2365
|
+
elif method == "cubic":
|
|
2366
|
+
X = np.column_stack([np.ones_like(x_arr), x_arr, x_arr ** 2, x_arr ** 3])
|
|
2367
|
+
model = OLS(y_arr, X).fit()
|
|
2368
|
+
link_label = "cubic (degree-3)"
|
|
2369
|
+
elif method == "logistic":
|
|
2370
|
+
# Validate binary {0, 1}; fall back to OLS with a warning if not.
|
|
2371
|
+
unique_y = np.unique(y_arr[~np.isnan(y_arr)])
|
|
2372
|
+
is_binary = set(unique_y.tolist()).issubset({0.0, 1.0}) and len(unique_y) == 2
|
|
2373
|
+
if not is_binary:
|
|
2374
|
+
warnings.warn(
|
|
2375
|
+
f"method='logistic' requires a numeric binary 0/1 outcome; "
|
|
2376
|
+
f"{y!r} has unique values {sorted(unique_y.tolist())}. "
|
|
2377
|
+
f"Falling back to OLS.",
|
|
2378
|
+
UserWarning,
|
|
2379
|
+
stacklevel=3,
|
|
2380
|
+
)
|
|
2381
|
+
X = sm.add_constant(x_arr)
|
|
2382
|
+
model = OLS(y_arr, X).fit()
|
|
2383
|
+
link_label = "OLS fallback (logistic requires binary y)"
|
|
2384
|
+
else:
|
|
2385
|
+
X = sm.add_constant(x_arr)
|
|
2386
|
+
model = sm.GLM(
|
|
2387
|
+
y_arr, X, family=sm.families.Binomial(link=sm.families.links.Logit())
|
|
2388
|
+
).fit()
|
|
2389
|
+
link_label = "logistic (logit)"
|
|
2390
|
+
elif method == "rlm":
|
|
2391
|
+
X = sm.add_constant(x_arr)
|
|
2392
|
+
model = sm.RLM(y_arr, X, M=sm.robust.norms.HuberT()).fit()
|
|
2393
|
+
link_label = "rlm (Huber)"
|
|
2394
|
+
elif method == "poisson":
|
|
2395
|
+
X = sm.add_constant(x_arr)
|
|
2396
|
+
if np.any(y_arr < 0):
|
|
2397
|
+
warnings.warn(
|
|
2398
|
+
f"method='poisson' requires a non-negative outcome; {y!r} has "
|
|
2399
|
+
f"negative values. Falling back to OLS.",
|
|
2400
|
+
UserWarning,
|
|
2401
|
+
stacklevel=3,
|
|
2402
|
+
)
|
|
2403
|
+
model = OLS(y_arr, X).fit()
|
|
2404
|
+
link_label = "OLS fallback (poisson requires non-negative y)"
|
|
2405
|
+
else:
|
|
2406
|
+
model = sm.GLM(
|
|
2407
|
+
y_arr, X, family=sm.families.Poisson(link=sm.families.links.Log())
|
|
2408
|
+
).fit()
|
|
2409
|
+
link_label = "poisson (log)"
|
|
2410
|
+
elif method == "Gamma":
|
|
2411
|
+
X = sm.add_constant(x_arr)
|
|
2412
|
+
if np.any(y_arr <= 0):
|
|
2413
|
+
warnings.warn(
|
|
2414
|
+
f"method='Gamma' requires a strictly positive outcome; {y!r} has "
|
|
2415
|
+
f"non-positive values. Falling back to OLS.",
|
|
2416
|
+
UserWarning,
|
|
2417
|
+
stacklevel=3,
|
|
2418
|
+
)
|
|
2419
|
+
model = OLS(y_arr, X).fit()
|
|
2420
|
+
link_label = "OLS fallback (Gamma requires positive y)"
|
|
2421
|
+
else:
|
|
2422
|
+
model = sm.GLM(
|
|
2423
|
+
y_arr, X,
|
|
2424
|
+
family=sm.families.Gamma(link=sm.families.links.InversePower()),
|
|
2425
|
+
).fit()
|
|
2426
|
+
link_label = "Gamma (inverse)"
|
|
2427
|
+
elif method in {"mixedlm", "lmer"}:
|
|
2428
|
+
if mixed_backend not in {"auto", "statsmodels"}:
|
|
2429
|
+
raise ValueError(
|
|
2430
|
+
f"mixed_backend must be 'auto' or 'statsmodels'; got {mixed_backend!r}."
|
|
2431
|
+
)
|
|
2432
|
+
re_formula, group_col = _parse_random_effects_spec(random_effects, x)
|
|
2433
|
+
fit_df = pd.DataFrame({"y": y_arr, "x": x_arr, "__group": data[group_col]})
|
|
2434
|
+
try:
|
|
2435
|
+
model = smf.mixedlm(
|
|
2436
|
+
"y ~ x",
|
|
2437
|
+
data=fit_df,
|
|
2438
|
+
groups=fit_df["__group"],
|
|
2439
|
+
re_formula=re_formula,
|
|
2440
|
+
).fit(reml=True, method="lbfgs", disp=False)
|
|
2441
|
+
except Exception as exc:
|
|
2442
|
+
raise RuntimeError(
|
|
2443
|
+
f"Failed to fit mixedlm/lmer model: {exc}"
|
|
2444
|
+
) from exc
|
|
2445
|
+
link_label = "mixedlm/lmer"
|
|
2446
|
+
elif method == "glmer":
|
|
2447
|
+
if mixed_backend not in {"auto", "statsmodels"}:
|
|
2448
|
+
raise ValueError(
|
|
2449
|
+
f"mixed_backend must be 'auto' or 'statsmodels'; got {mixed_backend!r}."
|
|
2450
|
+
)
|
|
2451
|
+
unique_y = np.unique(y_arr[~np.isnan(y_arr)])
|
|
2452
|
+
is_binary = set(unique_y.tolist()).issubset({0.0, 1.0}) and len(unique_y) == 2
|
|
2453
|
+
if not is_binary:
|
|
2454
|
+
raise ValueError(
|
|
2455
|
+
f"method='glmer' requires a numeric binary 0/1 outcome; "
|
|
2456
|
+
f"{y!r} has unique values {sorted(unique_y.tolist())}."
|
|
2457
|
+
)
|
|
2458
|
+
_, group_col = _parse_random_effects_spec(random_effects, x)
|
|
2459
|
+
fit_df = pd.DataFrame({"y": y_arr, "x": x_arr, "__group": data[group_col]})
|
|
2460
|
+
from statsmodels.genmod.bayes_mixed_glm import BinomialBayesMixedGLM
|
|
2461
|
+
try:
|
|
2462
|
+
model = BinomialBayesMixedGLM.from_formula(
|
|
2463
|
+
"y ~ x",
|
|
2464
|
+
{"__group": "0 + C(__group)"},
|
|
2465
|
+
fit_df,
|
|
2466
|
+
).fit_vb()
|
|
2467
|
+
except Exception as exc:
|
|
2468
|
+
raise RuntimeError(f"Failed to fit glmer model: {exc}") from exc
|
|
2469
|
+
link_label = "glmer"
|
|
2470
|
+
else: # pragma: no cover — guarded by caller
|
|
2471
|
+
return p
|
|
2472
|
+
|
|
2473
|
+
x_eval = np.linspace(np.nanmin(x_arr), np.nanmax(x_arr), num=200)
|
|
2474
|
+
|
|
2475
|
+
# Build the corresponding evaluation design matrix.
|
|
2476
|
+
if method in {"quadratic", "polynomial"}:
|
|
2477
|
+
X_eval = np.column_stack([np.ones_like(x_eval), x_eval, x_eval ** 2])
|
|
2478
|
+
elif method == "cubic":
|
|
2479
|
+
X_eval = np.column_stack(
|
|
2480
|
+
[np.ones_like(x_eval), x_eval, x_eval ** 2, x_eval ** 3]
|
|
2481
|
+
)
|
|
2482
|
+
elif method in {"logistic", "rlm", "poisson", "Gamma"}:
|
|
2483
|
+
# logistic, rlm, poisson, Gamma (and logistic OLS fallback) are all
|
|
2484
|
+
# linear-in-x models: intercept + x.
|
|
2485
|
+
X_eval = sm.add_constant(x_eval)
|
|
2486
|
+
elif method in {"mixedlm", "lmer"}:
|
|
2487
|
+
X_eval = np.column_stack([np.ones_like(x_eval), x_eval])
|
|
2488
|
+
elif method == "glmer":
|
|
2489
|
+
X_eval = np.column_stack([np.ones_like(x_eval), x_eval])
|
|
2490
|
+
else: # pragma: no cover
|
|
2491
|
+
X_eval = sm.add_constant(x_eval)
|
|
2492
|
+
|
|
2493
|
+
if method in {"mixedlm", "lmer"}:
|
|
2494
|
+
fe_names = list(model.fe_params.index)
|
|
2495
|
+
fe_mean = np.asarray(model.fe_params.to_numpy(), dtype=float)
|
|
2496
|
+
cov_df = model.cov_params()
|
|
2497
|
+
if hasattr(cov_df, "loc"):
|
|
2498
|
+
fe_cov = np.asarray(cov_df.loc[fe_names, fe_names], dtype=float)
|
|
2499
|
+
else:
|
|
2500
|
+
fe_cov = np.asarray(cov_df, dtype=float)[: len(fe_names), : len(fe_names)]
|
|
2501
|
+
design = np.column_stack([np.ones_like(x_eval), x_eval])
|
|
2502
|
+
yhat_eval = design @ fe_mean
|
|
2503
|
+
ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
|
|
2504
|
+
for layer in _add_fe_band(ribbon_df, design, fe_cov):
|
|
2505
|
+
p += layer
|
|
2506
|
+
p += geom_line(aes(y=y), data=ribbon_df, color="blue", inherit_aes=False)
|
|
2507
|
+
_ = link_label
|
|
2508
|
+
return p
|
|
2509
|
+
if method == "glmer":
|
|
2510
|
+
beta = np.asarray(model.fe_mean, dtype=float)
|
|
2511
|
+
design = np.column_stack([np.ones_like(x_eval), x_eval])
|
|
2512
|
+
eta = design @ beta
|
|
2513
|
+
yhat_eval = 1.0 / (1.0 + np.exp(-eta))
|
|
2514
|
+
ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
|
|
2515
|
+
# Approximate FE-only bands from posterior SD; this ignores covariance,
|
|
2516
|
+
# but gives a stable uncertainty envelope without requiring MCMC draws.
|
|
2517
|
+
if bands is not None:
|
|
2518
|
+
levels = sorted(set(bands))
|
|
2519
|
+
else:
|
|
2520
|
+
levels = [level]
|
|
2521
|
+
fe_sd = np.asarray(getattr(model, "fe_sd", np.full_like(beta, np.nan)), dtype=float)
|
|
2522
|
+
se_eta = np.sqrt(np.maximum((design ** 2) @ (fe_sd ** 2), 0.0))
|
|
2523
|
+
for lvl in levels:
|
|
2524
|
+
z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
|
|
2525
|
+
lo = 1.0 / (1.0 + np.exp(-(eta - z * se_eta)))
|
|
2526
|
+
hi = 1.0 / (1.0 + np.exp(-(eta + z * se_eta)))
|
|
2527
|
+
ribbon_df[f"__lower_{lvl}"] = lo
|
|
2528
|
+
ribbon_df[f"__upper_{lvl}"] = hi
|
|
2529
|
+
for lvl in sorted(levels, reverse=True):
|
|
2530
|
+
alpha = 0.1 + 0.15 * (lvl / max(levels))
|
|
2531
|
+
p += geom_ribbon(
|
|
2532
|
+
aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
|
|
2533
|
+
data=ribbon_df,
|
|
2534
|
+
alpha=alpha,
|
|
2535
|
+
fill="blue",
|
|
2536
|
+
inherit_aes=False,
|
|
2537
|
+
)
|
|
2538
|
+
p += geom_line(aes(y=y), data=ribbon_df, color="blue", inherit_aes=False)
|
|
2539
|
+
_ = link_label
|
|
2540
|
+
return p
|
|
2541
|
+
|
|
2542
|
+
yhat_eval = np.asarray(model.predict(X_eval))
|
|
2543
|
+
|
|
2544
|
+
# --- Bands: nested or single ---
|
|
2545
|
+
if bands is not None:
|
|
2546
|
+
levels = sorted(set(bands))
|
|
2547
|
+
else:
|
|
2548
|
+
levels = [level]
|
|
2549
|
+
|
|
2550
|
+
# Outermost band draws the line; inner bands draw only the ribbon.
|
|
2551
|
+
# Build a single combined ribbon dataframe with all band columns so we
|
|
2552
|
+
# can layer them on the same plot. Outermost band = widest.
|
|
2553
|
+
ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
|
|
2554
|
+
for lvl in levels:
|
|
2555
|
+
z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
|
|
2556
|
+
# Use the prediction SE for the mean (not for new observations) for
|
|
2557
|
+
# a CI-style band. statsmodels' get_prediction().summary_frame(alpha)
|
|
2558
|
+
# gives mean_ci_lower / mean_ci_upper directly.
|
|
2559
|
+
try:
|
|
2560
|
+
pred = model.get_prediction(X_eval)
|
|
2561
|
+
frame = pred.summary_frame(alpha=1 - lvl)
|
|
2562
|
+
lower = frame["mean_ci_lower"].to_numpy()
|
|
2563
|
+
upper = frame["mean_ci_upper"].to_numpy()
|
|
2564
|
+
except Exception:
|
|
2565
|
+
# Fallback: normal-approx CI using the model's parameter covariance.
|
|
2566
|
+
# For RLM, get_prediction() is not available, so we compute
|
|
2567
|
+
# Var(x' beta) = diag(X_eval @ Cov(beta) @ X_eval').
|
|
2568
|
+
cov_params = getattr(model, "cov_params", None)
|
|
2569
|
+
if cov_params is not None:
|
|
2570
|
+
try:
|
|
2571
|
+
var = np.einsum("ij,jk,ik->i", X_eval, cov_params(), X_eval)
|
|
2572
|
+
se = np.sqrt(var)
|
|
2573
|
+
except Exception:
|
|
2574
|
+
se = np.full(len(yhat_eval), np.nan)
|
|
2575
|
+
else:
|
|
2576
|
+
se = np.full(len(yhat_eval), np.nan)
|
|
2577
|
+
# Guard degenerate SE (e.g., perfect fit); draw a flat band.
|
|
2578
|
+
finite_se = np.where(np.isfinite(se), se, 0.0)
|
|
2579
|
+
lower = yhat_eval - z * finite_se
|
|
2580
|
+
upper = yhat_eval + z * finite_se
|
|
2581
|
+
|
|
2582
|
+
ribbon_df[f"__lower_{lvl}"] = lower
|
|
2583
|
+
ribbon_df[f"__upper_{lvl}"] = upper
|
|
2584
|
+
|
|
2585
|
+
# Draw ribbons (innermost first so outermost ends up on top).
|
|
2586
|
+
for lvl in sorted(levels, reverse=True):
|
|
2587
|
+
alpha = 0.1 + 0.15 * (lvl / max(levels))
|
|
2588
|
+
p += geom_ribbon(
|
|
2589
|
+
aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
|
|
2590
|
+
data=ribbon_df,
|
|
2591
|
+
alpha=alpha,
|
|
2592
|
+
fill="blue",
|
|
2593
|
+
inherit_aes=False,
|
|
2594
|
+
)
|
|
2595
|
+
|
|
2596
|
+
p += geom_line(
|
|
2597
|
+
aes(y=y),
|
|
2598
|
+
data=ribbon_df,
|
|
2599
|
+
color="blue",
|
|
2600
|
+
inherit_aes=False,
|
|
2601
|
+
)
|
|
2602
|
+
|
|
2603
|
+
# Inject the link label into the plot's labels so users can see what
|
|
2604
|
+
# was fit. plotnine exposes .labels; mutate via a workaround (geom_line
|
|
2605
|
+
# doesn't carry labels, so attach as a one-off text annotation is
|
|
2606
|
+
# cleaner — but text annotations need positioning data. Skipping for
|
|
2607
|
+
# now; users can pass `labs()` themselves).
|
|
2608
|
+
_ = link_label # reserved for future annotation hook
|
|
2609
|
+
return p
|
|
2610
|
+
|
|
2611
|
+
|
|
2612
|
+
def _add_interaction_smooth(
|
|
2613
|
+
p,
|
|
2614
|
+
data: pd.DataFrame,
|
|
2615
|
+
x: str,
|
|
2616
|
+
y: str,
|
|
2617
|
+
color: str,
|
|
2618
|
+
all_x: list,
|
|
2619
|
+
method: str,
|
|
2620
|
+
uncertainty: Optional[str],
|
|
2621
|
+
level: float,
|
|
2622
|
+
bands: Optional[List[float]],
|
|
2623
|
+
):
|
|
2624
|
+
"""Add per-color-group fitted lines for an interaction formula.
|
|
2625
|
+
|
|
2626
|
+
Used when ``interaction_model=True`` and the formula contains ``*`` or
|
|
2627
|
+
``:`` syntax. Fits a statsmodels OLS with the actual interaction term
|
|
2628
|
+
(e.g. ``y ~ x * z`` rather than ``y ~ x + z``) and overlays one
|
|
2629
|
+
``geom_line`` + optional ``geom_ribbon`` per level of ``color``.
|
|
2630
|
+
|
|
2631
|
+
Parameters
|
|
2632
|
+
----------
|
|
2633
|
+
all_x : list of str
|
|
2634
|
+
The expanded predictor list from the parser; for ``y ~ x*z`` this
|
|
2635
|
+
is ``['x', 'z', 'x:z']``. The interaction term is auto-detected as
|
|
2636
|
+
any term containing ``:``.
|
|
2637
|
+
"""
|
|
2638
|
+
# Identify the interaction term in all_x (the one containing ":").
|
|
2639
|
+
interaction_term = next((t for t in all_x if ":" in t), None)
|
|
2640
|
+
if interaction_term is None:
|
|
2641
|
+
# Defensive: if interaction_model=True but no interaction term
|
|
2642
|
+
# was parsed, fall back to the additive path. (Should not happen
|
|
2643
|
+
# if the parser is consistent, but we don't want to crash.)
|
|
2644
|
+
return _add_numeric_smooth(
|
|
2645
|
+
p, data, x, y, method, uncertainty, level, bands
|
|
2646
|
+
)
|
|
2647
|
+
|
|
2648
|
+
# Find the color column name in the interaction term: "x:z" -> first atom.
|
|
2649
|
+
color_atom = _first_atom(interaction_term.split(":")[1]) if ":" in interaction_term else color
|
|
2650
|
+
if color_atom != color:
|
|
2651
|
+
# Mismatch — fallback.
|
|
2652
|
+
return _add_numeric_smooth(
|
|
2653
|
+
p, data, x, y, method, uncertainty, level, bands
|
|
2654
|
+
)
|
|
2655
|
+
|
|
2656
|
+
# Build the design matrix: y ~ x + color + x:color (the interaction).
|
|
2657
|
+
x_arr = data[x].to_numpy(dtype=float)
|
|
2658
|
+
color_arr = data[color].to_numpy()
|
|
2659
|
+
y_arr = data[y].to_numpy(dtype=float)
|
|
2660
|
+
|
|
2661
|
+
# Encode color via a category code so the interaction term is numeric.
|
|
2662
|
+
color_series = pd.Series(color_arr)
|
|
2663
|
+
color_codes, color_levels = pd.factorize(color_series)
|
|
2664
|
+
color_codes = color_codes.astype(float)
|
|
2665
|
+
n_groups = len(color_levels)
|
|
2666
|
+
if n_groups < 2:
|
|
2667
|
+
# Degenerate: only one color level. Fall back to additive.
|
|
2668
|
+
return _add_numeric_smooth(
|
|
2669
|
+
p, data, x, y, method, uncertainty, level, bands
|
|
2670
|
+
)
|
|
2671
|
+
|
|
2672
|
+
# Build design matrix: intercept + x + color_dummies (drop first) + x:color_dummies
|
|
2673
|
+
# Simpler approach: use statsmodels' formula API directly with the
|
|
2674
|
+
# interaction term. This is cleaner than constructing the design
|
|
2675
|
+
# matrix by hand and matches R's `y ~ x * color` semantics.
|
|
2676
|
+
import statsmodels.formula.api as _smf
|
|
2677
|
+
|
|
2678
|
+
# Build a temporary DataFrame for statsmodels.
|
|
2679
|
+
fit_df = pd.DataFrame({
|
|
2680
|
+
"_y": y_arr,
|
|
2681
|
+
"_x": x_arr,
|
|
2682
|
+
"_color": color_arr,
|
|
2683
|
+
})
|
|
2684
|
+
# Renaming so statsmodels' patsy accepts them (avoid operator parsing issues).
|
|
2685
|
+
fit_df.columns = ["_y", "_x", "_color"]
|
|
2686
|
+
formula_str = "_y ~ _x * _color"
|
|
2687
|
+
try:
|
|
2688
|
+
model = _smf.ols(formula_str, data=fit_df).fit()
|
|
2689
|
+
except Exception as exc:
|
|
2690
|
+
raise RuntimeError(
|
|
2691
|
+
f"interaction_model=True requires a valid OLS fit with the "
|
|
2692
|
+
f"interaction term; got: {exc}"
|
|
2693
|
+
) from exc
|
|
2694
|
+
|
|
2695
|
+
# Predict on a per-color grid.
|
|
2696
|
+
x_min = float(np.nanmin(x_arr))
|
|
2697
|
+
x_max = float(np.nanmax(x_arr))
|
|
2698
|
+
x_eval = np.linspace(x_min, x_max, num=200)
|
|
2699
|
+
|
|
2700
|
+
# Build eval DataFrame with one row per (x_eval, color_level).
|
|
2701
|
+
eval_rows = []
|
|
2702
|
+
for level_val in color_levels:
|
|
2703
|
+
for xv in x_eval:
|
|
2704
|
+
eval_rows.append({"_x": xv, "_color": level_val})
|
|
2705
|
+
eval_df = pd.DataFrame(eval_rows)
|
|
2706
|
+
yhat_eval = np.asarray(model.predict(eval_df))
|
|
2707
|
+
|
|
2708
|
+
# Compute CI bands. Supports a single level (level=) or nested bands
|
|
2709
|
+
# (bands=[...]). Returns a dict {level_value: (lower_array, upper_array)}
|
|
2710
|
+
# or None if CI computation failed / is suppressed.
|
|
2711
|
+
band_arrays: Optional[dict[float, tuple[Any, Any]]] = None
|
|
2712
|
+
if uncertainty in {"ci", "prediction"}:
|
|
2713
|
+
levels_to_compute = sorted(set(bands)) if bands is not None else [level]
|
|
2714
|
+
band_arrays = {}
|
|
2715
|
+
ci_kind = "obs_ci" if uncertainty == "prediction" else "mean_ci"
|
|
2716
|
+
for lvl in levels_to_compute:
|
|
2717
|
+
try:
|
|
2718
|
+
pred = model.get_prediction(eval_df)
|
|
2719
|
+
frame = pred.summary_frame(alpha=1 - lvl)
|
|
2720
|
+
lower = frame[f"{ci_kind}_lower"].to_numpy()
|
|
2721
|
+
upper = frame[f"{ci_kind}_upper"].to_numpy()
|
|
2722
|
+
band_arrays[lvl] = (lower, upper)
|
|
2723
|
+
except Exception:
|
|
2724
|
+
# Skip this level if statsmodels can't produce it.
|
|
2725
|
+
pass
|
|
2726
|
+
if not band_arrays:
|
|
2727
|
+
band_arrays = None
|
|
2728
|
+
|
|
2729
|
+
# Determine colors per group using a default palette.
|
|
2730
|
+
palette = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"]
|
|
2731
|
+
for i, level_val in enumerate(color_levels):
|
|
2732
|
+
group_mask = (eval_df["_color"] == level_val).to_numpy()
|
|
2733
|
+
line_color = palette[i % len(palette)]
|
|
2734
|
+
# Build a per-group dataframe for plotnine.
|
|
2735
|
+
line_df = pd.DataFrame({
|
|
2736
|
+
x: x_eval,
|
|
2737
|
+
y: yhat_eval[group_mask],
|
|
2738
|
+
color: np.full_like(x_eval, level_val, dtype=object),
|
|
2739
|
+
})
|
|
2740
|
+
# The line itself.
|
|
2741
|
+
p += geom_line(
|
|
2742
|
+
data=line_df,
|
|
2743
|
+
color=line_color,
|
|
2744
|
+
inherit_aes=False,
|
|
2745
|
+
)
|
|
2746
|
+
# Optional ribbons. For nested bands, draw innermost first so the
|
|
2747
|
+
# outermost band ends up on top.
|
|
2748
|
+
if band_arrays is not None:
|
|
2749
|
+
for lvl, (ci_lower, ci_upper) in sorted(band_arrays.items()):
|
|
2750
|
+
ribbon_df = line_df.copy()
|
|
2751
|
+
ribbon_df["__lower"] = ci_lower[group_mask]
|
|
2752
|
+
ribbon_df["__upper"] = ci_upper[group_mask]
|
|
2753
|
+
# Outer (larger coverage) bands are wider; lower alpha for
|
|
2754
|
+
# the innermost so the layering reads as Tufte-style nested.
|
|
2755
|
+
alpha = 0.10 + 0.10 * (lvl / max(band_arrays.keys()))
|
|
2756
|
+
p += geom_ribbon(
|
|
2757
|
+
aes(ymin="__lower", ymax="__upper"),
|
|
2758
|
+
data=ribbon_df,
|
|
2759
|
+
alpha=alpha,
|
|
2760
|
+
fill=line_color,
|
|
2761
|
+
inherit_aes=False,
|
|
2762
|
+
)
|
|
2763
|
+
return p
|
|
2764
|
+
|
|
2765
|
+
|
|
2766
|
+
def _add_binomial_smooth(
|
|
2767
|
+
p,
|
|
2768
|
+
data: pd.DataFrame,
|
|
2769
|
+
x: str,
|
|
2770
|
+
y: str,
|
|
2771
|
+
uncertainty: Optional[str],
|
|
2772
|
+
level: float,
|
|
2773
|
+
bands: Optional[List[float]],
|
|
2774
|
+
color: Optional[str] = None,
|
|
2775
|
+
):
|
|
2776
|
+
"""Add fitted line + uncertainty band for numeric-vs-binary (binomial GLM).
|
|
2777
|
+
|
|
2778
|
+
The band is always rendered on the response (probability) scale.
|
|
2779
|
+
"""
|
|
2780
|
+
if uncertainty is None:
|
|
2781
|
+
# No fit at all — just the scatter.
|
|
2782
|
+
return p
|
|
2783
|
+
|
|
2784
|
+
method_args = {"family": "binomial"}
|
|
2785
|
+
line_color = None if color else "blue"
|
|
2786
|
+
|
|
2787
|
+
if bands is not None:
|
|
2788
|
+
levels = sorted(set(bands))
|
|
2789
|
+
for lvl in levels:
|
|
2790
|
+
kwargs = {
|
|
2791
|
+
"method": "glm",
|
|
2792
|
+
"method_args": method_args,
|
|
2793
|
+
"level": lvl,
|
|
2794
|
+
"alpha": 0.15,
|
|
2795
|
+
}
|
|
2796
|
+
if line_color:
|
|
2797
|
+
kwargs["color"] = line_color
|
|
2798
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2799
|
+
return p
|
|
2800
|
+
|
|
2801
|
+
kwargs = {
|
|
2802
|
+
"method": "glm",
|
|
2803
|
+
"method_args": method_args,
|
|
2804
|
+
"level": level,
|
|
2805
|
+
}
|
|
2806
|
+
if line_color:
|
|
2807
|
+
kwargs["color"] = line_color
|
|
2808
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2809
|
+
return p
|
|
2810
|
+
|
|
2811
|
+
|
|
2812
|
+
def _add_overlay_numeric(p, data, x, y, overlay_specs):
|
|
2813
|
+
"""Add one geom_smooth per overlay spec on the numeric-vs-numeric branch.
|
|
2814
|
+
|
|
2815
|
+
Each entry is drawn with its own color and ``uncertainty``/``level``
|
|
2816
|
+
settings. No bootstrap overlay for non-loess methods (plotnine doesn't
|
|
2817
|
+
expose stat_smooth's bootstrap from a single ``geom_smooth`` call with
|
|
2818
|
+
arbitrary methods; we keep the API consistent by routing all overlays
|
|
2819
|
+
through ``geom_smooth`` and reserving bootstrap for ``"loess"``).
|
|
2820
|
+
"""
|
|
2821
|
+
has_labels = any(
|
|
2822
|
+
spec.get("label") and spec["label"] != spec["method"]
|
|
2823
|
+
for spec in overlay_specs
|
|
2824
|
+
)
|
|
2825
|
+
label_colors = {}
|
|
2826
|
+
|
|
2827
|
+
for spec in overlay_specs:
|
|
2828
|
+
method = spec["method"]
|
|
2829
|
+
level = spec["level"]
|
|
2830
|
+
color = spec["color"]
|
|
2831
|
+
label = spec.get("label", method)
|
|
2832
|
+
kwargs = {"method": method, "level": level, "color": color}
|
|
2833
|
+
# Forward any extra stat_args (span, formula, method_args, ...) that
|
|
2834
|
+
# the user provided.
|
|
2835
|
+
for k in ("span", "formula", "method_args", "n"):
|
|
2836
|
+
if k in spec:
|
|
2837
|
+
kwargs[k] = spec[k]
|
|
2838
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2839
|
+
if has_labels:
|
|
2840
|
+
label_colors[label] = color
|
|
2841
|
+
|
|
2842
|
+
if has_labels and label_colors:
|
|
2843
|
+
# Add a manual color scale so labels appear in the legend.
|
|
2844
|
+
p += scale_color_manual(
|
|
2845
|
+
name="Method",
|
|
2846
|
+
values=label_colors,
|
|
2847
|
+
)
|
|
2848
|
+
return p
|
|
2849
|
+
|
|
2850
|
+
|
|
2851
|
+
def _add_overlay_binomial(p, data, x, y, overlay_specs):
|
|
2852
|
+
"""Add binomial GLM overlay smoothers on the numeric-vs-binary branch.
|
|
2853
|
+
|
|
2854
|
+
Only entries with ``method == "glm"`` are supported here; other methods
|
|
2855
|
+
raise so the user gets a clear error rather than a silently-broken chart.
|
|
2856
|
+
"""
|
|
2857
|
+
label_colors = {}
|
|
2858
|
+
for spec in overlay_specs:
|
|
2859
|
+
if spec["method"] != "glm":
|
|
2860
|
+
raise ValueError(
|
|
2861
|
+
f"Overlay method {spec['method']!r} is not supported on the "
|
|
2862
|
+
f"binomial branch; only 'glm' is allowed."
|
|
2863
|
+
)
|
|
2864
|
+
kwargs = {
|
|
2865
|
+
"method": "glm",
|
|
2866
|
+
"method_args": {"family": "binomial"},
|
|
2867
|
+
"level": spec["level"],
|
|
2868
|
+
"color": spec["color"],
|
|
2869
|
+
}
|
|
2870
|
+
p += cast(Any, geom_smooth)(**kwargs)
|
|
2871
|
+
label_colors[spec.get("label", "glm")] = spec["color"]
|
|
2872
|
+
if label_colors:
|
|
2873
|
+
p += scale_color_manual(name="Method", values=label_colors)
|
|
2874
|
+
return p
|
|
2875
|
+
|
|
2876
|
+
|
|
2877
|
+
def _first_non_intercept_name(model, fallback="x"):
|
|
2878
|
+
"""Return the first non-intercept exog name or None if only intercept."""
|
|
2879
|
+
names = getattr(getattr(model, "model", None), "exog_names", None) or []
|
|
2880
|
+
non_intercept = [n for n in names if n not in ("Intercept", "const")]
|
|
2881
|
+
return non_intercept[0] if non_intercept else None
|
|
2882
|
+
|
|
2883
|
+
|
|
2884
|
+
def _is_neural_net_fit(model) -> bool:
|
|
2885
|
+
"""Duck-type detection of a :class:`pyflexplot.flex_nn.NeuralNetFit`.
|
|
2886
|
+
|
|
2887
|
+
Avoids importing :mod:`flex_nn` at module load time so the core module
|
|
2888
|
+
stays cheap to import when neural-net support isn't needed. A
|
|
2889
|
+
``NeuralNetFit`` exposes ``.predict(data)`` returning an indexed
|
|
2890
|
+
Series plus the response-var metadata the wrapper carries.
|
|
2891
|
+
"""
|
|
2892
|
+
cls = type(model)
|
|
2893
|
+
cls_name = f"{cls.__module__}.{cls.__qualname__}"
|
|
2894
|
+
return cls_name == "pyflexplot.flex_nn.NeuralNetFit"
|
|
2895
|
+
|
|
2896
|
+
|
|
2897
|
+
def _visualize_neural_net(fit, data=None, **kwargs):
|
|
2898
|
+
"""Visualization path for ``NeuralNetFit`` wrappers.
|
|
2899
|
+
|
|
2900
|
+
Mirrors the statsmodels ``visualize()`` output (predicted-vs-actual
|
|
2901
|
+
line on top of a scatter) so a user can drop a fitted network into
|
|
2902
|
+
the same plot they would have used for an OLS fit.
|
|
2903
|
+
"""
|
|
2904
|
+
if data is None:
|
|
2905
|
+
raise ValueError(
|
|
2906
|
+
"visualize() requires data= when called on a NeuralNetFit "
|
|
2907
|
+
"(the wrapper does not carry the training data)."
|
|
2908
|
+
)
|
|
2909
|
+
if not isinstance(data, pd.DataFrame):
|
|
2910
|
+
raise TypeError(
|
|
2911
|
+
f"data must be a pandas DataFrame, got {type(data).__name__}"
|
|
2912
|
+
)
|
|
2913
|
+
if data.empty:
|
|
2914
|
+
raise ValueError("data must be non-empty for visualization.")
|
|
2915
|
+
|
|
2916
|
+
response_var = fit.response_var
|
|
2917
|
+
if response_var not in data.columns:
|
|
2918
|
+
raise ValueError(
|
|
2919
|
+
f"Response column {response_var!r} (declared on the NeuralNetFit) "
|
|
2920
|
+
f"not found in data. Available columns: {list(data.columns)}"
|
|
2921
|
+
)
|
|
2922
|
+
|
|
2923
|
+
# Determine the x predictor. Honour explicit x=, otherwise use the
|
|
2924
|
+
# first declared predictor (mirrors the statsmodels fallback in
|
|
2925
|
+
# visualize()).
|
|
2926
|
+
x_name = kwargs.get("x")
|
|
2927
|
+
if x_name is None:
|
|
2928
|
+
if not fit.predictor_names:
|
|
2929
|
+
raise ValueError(
|
|
2930
|
+
"NeuralNetFit has no declared predictor_names; "
|
|
2931
|
+
"pass an explicit x= argument."
|
|
2932
|
+
)
|
|
2933
|
+
x_name = fit.predictor_names[0]
|
|
2934
|
+
if x_name not in data.columns:
|
|
2935
|
+
raise ValueError(
|
|
2936
|
+
f"Predictor column {x_name!r} not found in data for visualization."
|
|
2937
|
+
)
|
|
2938
|
+
|
|
2939
|
+
pred = fit.predict(data)
|
|
2940
|
+
|
|
2941
|
+
plot_df = data.copy()
|
|
2942
|
+
plot_df["__predicted"] = pred.reindex(plot_df.index)
|
|
2943
|
+
|
|
2944
|
+
p = (
|
|
2945
|
+
ggplot(plot_df, aes(x=x_name, y=response_var))
|
|
2946
|
+
+ geom_point(alpha=0.4)
|
|
2947
|
+
+ geom_line(aes(y="__predicted"), color="red", size=1)
|
|
2948
|
+
+ labs(
|
|
2949
|
+
title=f"Visualization: NeuralNetFit ({fit.backend})",
|
|
2950
|
+
subtitle=f"Predicted {response_var} vs {x_name}",
|
|
2951
|
+
)
|
|
2952
|
+
+ theme_bw()
|
|
2953
|
+
)
|
|
2954
|
+
return p
|
|
2955
|
+
|
|
2956
|
+
|
|
2957
|
+
def visualize(
|
|
2958
|
+
model,
|
|
2959
|
+
data: Optional[pd.DataFrame] = None,
|
|
2960
|
+
plot: str = "model",
|
|
2961
|
+
**kwargs,
|
|
2962
|
+
):
|
|
2963
|
+
"""
|
|
2964
|
+
Provides a visual representation of a fitted statistical object.
|
|
2965
|
+
Supports statsmodels (OLS, GLM), scikit-learn models, and
|
|
2966
|
+
:class:`pyflexplot.flex_nn.NeuralNetFit` wrappers.
|
|
2967
|
+
|
|
2968
|
+
Parameters
|
|
2969
|
+
----------
|
|
2970
|
+
model : fitted model
|
|
2971
|
+
Any model with a ``predict`` method. Statsmodels (OLS, GLM),
|
|
2972
|
+
scikit-learn regressors, and ``NeuralNetFit`` are supported.
|
|
2973
|
+
data : pd.DataFrame, optional
|
|
2974
|
+
Predictor data. If omitted, inferred from ``model.model.data``.
|
|
2975
|
+
plot : {"model", "residuals", "all"}, default "model"
|
|
2976
|
+
What to draw:
|
|
2977
|
+
- ``"model"``: predicted-vs-observed scatter with the fitted
|
|
2978
|
+
line (the legacy behavior).
|
|
2979
|
+
- ``"residuals"``: residual-vs-fitted scatter and a residual
|
|
2980
|
+
histogram, side by side. Mirrors R's
|
|
2981
|
+
``visualize.lm(plot="residuals")``.
|
|
2982
|
+
- ``"all"``: a combined ``cowplot``-style panel with the model
|
|
2983
|
+
fit on the left and the residual plots on the right.
|
|
2984
|
+
When ``cowplot`` isn't installed, the components are returned
|
|
2985
|
+
as a dict of named layers.
|
|
2986
|
+
|
|
2987
|
+
Returns
|
|
2988
|
+
-------
|
|
2989
|
+
plotnine.ggplot OR dict
|
|
2990
|
+
When ``plot="model"`` or ``plot="all"`` (with cowplot), a
|
|
2991
|
+
single ggplot (or cowplot-joined object). When
|
|
2992
|
+
``plot="residuals"``, a dict with ``"rvf"`` (residual-vs-fitted)
|
|
2993
|
+
and ``"hist"`` (residual histogram) ggplot objects.
|
|
2994
|
+
"""
|
|
2995
|
+
# NeuralNetFit path: duck-typed dispatch so core.py doesn't have to
|
|
2996
|
+
# import flex_nn at module load time.
|
|
2997
|
+
if _is_neural_net_fit(model):
|
|
2998
|
+
return _visualize_neural_net(model, data=data, **kwargs)
|
|
2999
|
+
|
|
3000
|
+
plot = str(plot).lower()
|
|
3001
|
+
valid_plots = ("model", "residuals", "all")
|
|
3002
|
+
if plot not in valid_plots:
|
|
3003
|
+
raise ValueError(
|
|
3004
|
+
f"plot must be one of {valid_plots}; got {plot!r}."
|
|
3005
|
+
)
|
|
3006
|
+
|
|
3007
|
+
if data is None:
|
|
3008
|
+
if hasattr(model, "model") and hasattr(model.model, "data"):
|
|
3009
|
+
data = pd.DataFrame(model.model.data.orig_endog).join(
|
|
3010
|
+
pd.DataFrame(model.model.data.orig_exog)
|
|
3011
|
+
)
|
|
3012
|
+
else:
|
|
3013
|
+
raise ValueError("No data provided for visualization.")
|
|
3014
|
+
|
|
3015
|
+
if not isinstance(data, pd.DataFrame):
|
|
3016
|
+
raise TypeError(f"data must be a pandas DataFrame, got {type(data).__name__}")
|
|
3017
|
+
if data.empty:
|
|
3018
|
+
raise ValueError("data must be non-empty for visualization.")
|
|
3019
|
+
|
|
3020
|
+
if not hasattr(model, "predict"):
|
|
3021
|
+
raise NotImplementedError(
|
|
3022
|
+
f"Visualization for {type(model).__name__} not yet implemented "
|
|
3023
|
+
"(model has no predict method)."
|
|
3024
|
+
)
|
|
3025
|
+
|
|
3026
|
+
endog_names = getattr(getattr(model, "model", None), "endog_names", None)
|
|
3027
|
+
if endog_names is None:
|
|
3028
|
+
raise ValueError("Cannot determine outcome variable name from model.")
|
|
3029
|
+
|
|
3030
|
+
# Statsmodels predict returns a Series aligned to the input index.
|
|
3031
|
+
y_pred = model.predict(data)
|
|
3032
|
+
if not hasattr(y_pred, "index"):
|
|
3033
|
+
y_pred = pd.Series(y_pred, index=data.index)
|
|
3034
|
+
|
|
3035
|
+
plot_df = data.copy()
|
|
3036
|
+
plot_df["__predicted"] = y_pred.reindex(plot_df.index)
|
|
3037
|
+
if endog_names in plot_df.columns:
|
|
3038
|
+
plot_df["__residual"] = plot_df[endog_names] - plot_df["__predicted"]
|
|
3039
|
+
else:
|
|
3040
|
+
plot_df["__residual"] = pd.Series(
|
|
3041
|
+
getattr(model, "resid", pd.Series(dtype=float))
|
|
3042
|
+
).reindex(plot_df.index)
|
|
3043
|
+
|
|
3044
|
+
# Determine the first predictor robustly.
|
|
3045
|
+
x_name = kwargs.get("x")
|
|
3046
|
+
if x_name is None:
|
|
3047
|
+
x_name = _first_non_intercept_name(model)
|
|
3048
|
+
if x_name is None:
|
|
3049
|
+
raise ValueError(
|
|
3050
|
+
"visualize() requires a non-intercept model or explicit x= argument."
|
|
3051
|
+
)
|
|
3052
|
+
if x_name not in plot_df.columns:
|
|
3053
|
+
raise ValueError(
|
|
3054
|
+
f"Predictor column {x_name!r} not found in data for visualization."
|
|
3055
|
+
)
|
|
3056
|
+
|
|
3057
|
+
# The "model" panel — legacy predicted-vs-observed plot.
|
|
3058
|
+
p_model = (
|
|
3059
|
+
ggplot(plot_df, aes(x=x_name, y=endog_names))
|
|
3060
|
+
+ geom_point(alpha=0.4)
|
|
3061
|
+
+ geom_line(aes(y="__predicted"), color="red", size=1)
|
|
3062
|
+
+ labs(
|
|
3063
|
+
title=f"Visualization: {type(model).__name__}",
|
|
3064
|
+
subtitle=f"Predicted {endog_names} vs {x_name}",
|
|
3065
|
+
)
|
|
3066
|
+
+ theme_bw()
|
|
3067
|
+
)
|
|
3068
|
+
|
|
3069
|
+
if plot == "model":
|
|
3070
|
+
return p_model
|
|
3071
|
+
|
|
3072
|
+
# Residual plots (used for plot="residuals" or plot="all").
|
|
3073
|
+
plot_resid_df = plot_df.dropna(subset=["__residual", "__predicted"])
|
|
3074
|
+
p_rvf = (
|
|
3075
|
+
ggplot(plot_resid_df, aes(x="__predicted", y="__residual"))
|
|
3076
|
+
+ geom_point(alpha=0.4)
|
|
3077
|
+
+ geom_hline(yintercept=0, linetype="dashed", color="gray")
|
|
3078
|
+
+ labs(
|
|
3079
|
+
title="Residuals vs Predicted",
|
|
3080
|
+
x="Predicted",
|
|
3081
|
+
y="Residual",
|
|
3082
|
+
)
|
|
3083
|
+
+ theme_bw()
|
|
3084
|
+
)
|
|
3085
|
+
p_hist = (
|
|
3086
|
+
ggplot(plot_resid_df, aes(x="__residual"))
|
|
3087
|
+
+ geom_histogram(bins=30, fill="steelblue", color="white")
|
|
3088
|
+
+ labs(
|
|
3089
|
+
title="Residual Distribution",
|
|
3090
|
+
x="Residual",
|
|
3091
|
+
y="Count",
|
|
3092
|
+
)
|
|
3093
|
+
+ theme_bw()
|
|
3094
|
+
)
|
|
3095
|
+
|
|
3096
|
+
if plot == "residuals":
|
|
3097
|
+
return {"rvf": p_rvf, "hist": p_hist}
|
|
3098
|
+
|
|
3099
|
+
# plot == "all" — try to compose with cowplot; otherwise return dict.
|
|
3100
|
+
try:
|
|
3101
|
+
import cowplot # type: ignore
|
|
3102
|
+
return cowplot.plot_grid(
|
|
3103
|
+
p_model, p_rvf, p_hist,
|
|
3104
|
+
ncol=2,
|
|
3105
|
+
rel_widths=(0.6, 0.4),
|
|
3106
|
+
)
|
|
3107
|
+
except ImportError:
|
|
3108
|
+
return {
|
|
3109
|
+
"model": p_model,
|
|
3110
|
+
"rvf": p_rvf,
|
|
3111
|
+
"hist": p_hist,
|
|
3112
|
+
}
|
|
3113
|
+
|
|
3114
|
+
|
|
3115
|
+
def compare_fits(
|
|
3116
|
+
formula: str,
|
|
3117
|
+
data: pd.DataFrame,
|
|
3118
|
+
model1,
|
|
3119
|
+
model2,
|
|
3120
|
+
labels: List[str] = ["Model 1", "Model 2"],
|
|
3121
|
+
return_preds: bool = False,
|
|
3122
|
+
pred_type: str = "response",
|
|
3123
|
+
report_se: bool = False,
|
|
3124
|
+
re: bool = False,
|
|
3125
|
+
num_points: Optional[int] = None,
|
|
3126
|
+
clusters: Optional[int] = None,
|
|
3127
|
+
**kwargs,
|
|
3128
|
+
):
|
|
3129
|
+
"""
|
|
3130
|
+
Visually compare the fit of two different models (statsmodels/sklearn).
|
|
3131
|
+
|
|
3132
|
+
Parameters
|
|
3133
|
+
----------
|
|
3134
|
+
formula, data, model1, model2, labels :
|
|
3135
|
+
Existing arguments; see prior releases.
|
|
3136
|
+
return_preds : bool, default False
|
|
3137
|
+
When True, return the prediction DataFrame instead of the plot.
|
|
3138
|
+
Columns: the formula's predictor(s), ``__y`` (the observed
|
|
3139
|
+
response), and ``__pred1`` / ``__pred2`` (each model's
|
|
3140
|
+
prediction). This matches R's ``compare.fits(..., return.preds=TRUE)``.
|
|
3141
|
+
pred_type : {"response", "link"}, default "response"
|
|
3142
|
+
Type of predictions passed to ``statsmodels`` for GLM models.
|
|
3143
|
+
``"response"`` returns the probability scale (default);
|
|
3144
|
+
``"link"`` returns the linear-predictor scale. Ignored for
|
|
3145
|
+
non-GLM models. Matches R's ``compare.fits(..., pred.type=...)``.
|
|
3146
|
+
report_se : bool, default False
|
|
3147
|
+
R-parity stub: when True, would include standard-error bands on
|
|
3148
|
+
prediction lines. Currently ignored; reserved for a future release.
|
|
3149
|
+
re : bool, default False
|
|
3150
|
+
R-parity stub: when True, would include random-effects predictions
|
|
3151
|
+
for mixed models. Currently ignored; reserved for a future release.
|
|
3152
|
+
num_points : int, optional
|
|
3153
|
+
R-parity stub: when set, would evaluate predictions on a grid of
|
|
3154
|
+
``num_points`` points spanning the x-range. Currently ignored;
|
|
3155
|
+
reserved for a future release.
|
|
3156
|
+
clusters : int, optional
|
|
3157
|
+
R-parity stub: when set, would cluster the prediction grid.
|
|
3158
|
+
Currently ignored; reserved for a future release.
|
|
3159
|
+
|
|
3160
|
+
Returns
|
|
3161
|
+
-------
|
|
3162
|
+
plotnine.ggplot OR pandas.DataFrame
|
|
3163
|
+
When ``return_preds=False``, the comparison plot. When
|
|
3164
|
+
``return_preds=True``, a DataFrame with observed and predicted
|
|
3165
|
+
values for both models.
|
|
3166
|
+
"""
|
|
3167
|
+
if report_se or re or num_points is not None or clusters is not None:
|
|
3168
|
+
warnings.warn(
|
|
3169
|
+
"compare_fits(): arguments report_se/re/num_points/clusters are "
|
|
3170
|
+
"accepted for R API parity but are currently no-ops in py-flexplot.",
|
|
3171
|
+
UserWarning,
|
|
3172
|
+
stacklevel=2,
|
|
3173
|
+
)
|
|
3174
|
+
|
|
3175
|
+
variables = parse_flexplot_formula(formula)
|
|
3176
|
+
_validate_data_for_plot(formula, data, variables)
|
|
3177
|
+
|
|
3178
|
+
y_name = variables["y"]
|
|
3179
|
+
x_name = variables["x"]
|
|
3180
|
+
|
|
3181
|
+
pred1 = _get_model_predictions(model1, data, pred_type=pred_type)
|
|
3182
|
+
pred2 = _get_model_predictions(model2, data, pred_type=pred_type)
|
|
3183
|
+
|
|
3184
|
+
if len(pred1) != len(data) or len(pred2) != len(data):
|
|
3185
|
+
raise ValueError(
|
|
3186
|
+
f"Model predictions must match data length ({len(data)}): "
|
|
3187
|
+
f"got {len(pred1)} and {len(pred2)}."
|
|
3188
|
+
)
|
|
3189
|
+
|
|
3190
|
+
plot_df = data.copy()
|
|
3191
|
+
plot_df["__m1"] = pred1
|
|
3192
|
+
plot_df["__m2"] = pred2
|
|
3193
|
+
|
|
3194
|
+
if return_preds:
|
|
3195
|
+
return plot_df
|
|
3196
|
+
|
|
3197
|
+
p = (
|
|
3198
|
+
ggplot(plot_df, aes(x=x_name, y=y_name))
|
|
3199
|
+
+ geom_point(alpha=0.3)
|
|
3200
|
+
+ geom_line(aes(y="__m1", color='"#3498db"'), size=1)
|
|
3201
|
+
+ geom_line(aes(y="__m2", color='"#e74c3c"'), size=1)
|
|
3202
|
+
+ scale_color_identity(
|
|
3203
|
+
guide="legend",
|
|
3204
|
+
name="Model",
|
|
3205
|
+
labels=labels,
|
|
3206
|
+
breaks=["#3498db", "#e74c3c"],
|
|
3207
|
+
)
|
|
3208
|
+
+ labs(title="Visual Model Comparison", x=x_name, y=y_name)
|
|
3209
|
+
+ theme_bw()
|
|
3210
|
+
)
|
|
3211
|
+
|
|
3212
|
+
return p
|
|
3213
|
+
|
|
3214
|
+
|
|
3215
|
+
def third_eye(*args, **kwargs):
|
|
3216
|
+
"""R API placeholder for ``third.eye``.
|
|
3217
|
+
|
|
3218
|
+
The R package exposes ``third.eye`` as a specialized 3-way interaction
|
|
3219
|
+
visualization. py-flexplot intentionally keeps this as a stub for now,
|
|
3220
|
+
so API discovery/parity tooling can detect the endpoint while behavior
|
|
3221
|
+
remains explicitly unimplemented.
|
|
3222
|
+
"""
|
|
3223
|
+
raise NotImplementedError(
|
|
3224
|
+
"third_eye() is not implemented in py-flexplot yet. "
|
|
3225
|
+
"Use flexplot(..., interaction_model=True) for interaction visuals."
|
|
3226
|
+
)
|
|
3227
|
+
|
|
3228
|
+
|
|
3229
|
+
def _get_model_predictions(
|
|
3230
|
+
model,
|
|
3231
|
+
data: pd.DataFrame,
|
|
3232
|
+
pred_type: str = "response",
|
|
3233
|
+
) -> pd.Series:
|
|
3234
|
+
"""Return a pandas Series of predictions aligned to data.index.
|
|
3235
|
+
|
|
3236
|
+
Parameters
|
|
3237
|
+
----------
|
|
3238
|
+
model : fitted model with ``predict`` method
|
|
3239
|
+
data : pd.DataFrame
|
|
3240
|
+
Predictor data.
|
|
3241
|
+
pred_type : {"response", "link"}, default "response"
|
|
3242
|
+
Passed through to ``statsmodels`` GLM ``predict``; controls
|
|
3243
|
+
whether the prediction is on the response or link scale.
|
|
3244
|
+
Ignored for non-GLM models.
|
|
3245
|
+
"""
|
|
3246
|
+
if hasattr(model, "predict"):
|
|
3247
|
+
try:
|
|
3248
|
+
# Statsmodels fitted models accept a DataFrame and return a Series
|
|
3249
|
+
# indexed by the (possibly reduced) observation index. GLM models
|
|
3250
|
+
# accept a ``linear`` kwarg to switch between response and link.
|
|
3251
|
+
if hasattr(model, "model") and getattr(model, "model", None) is not None:
|
|
3252
|
+
model_class = type(model).__name__.lower()
|
|
3253
|
+
if "glm" in model_class:
|
|
3254
|
+
pred = model.predict(data, linear=(pred_type == "link"))
|
|
3255
|
+
else:
|
|
3256
|
+
pred = model.predict(data)
|
|
3257
|
+
else:
|
|
3258
|
+
pred = model.predict(data)
|
|
3259
|
+
except Exception:
|
|
3260
|
+
# scikit-learn style: needs a 2-D array-like input.
|
|
3261
|
+
pred = model.predict(data.values)
|
|
3262
|
+
else:
|
|
3263
|
+
raise ValueError(
|
|
3264
|
+
f"Model of type {type(model).__name__} has no predict method."
|
|
3265
|
+
)
|
|
3266
|
+
|
|
3267
|
+
if isinstance(pred, pd.Series):
|
|
3268
|
+
return pred.reindex(data.index)
|
|
3269
|
+
|
|
3270
|
+
if isinstance(pred, pd.DataFrame):
|
|
3271
|
+
if pred.shape[1] == 1:
|
|
3272
|
+
pred = pred.iloc[:, 0]
|
|
3273
|
+
else:
|
|
3274
|
+
raise ValueError(
|
|
3275
|
+
f"Model predictions must be 1-D, got shape {pred.shape}"
|
|
3276
|
+
)
|
|
3277
|
+
return pred.reindex(data.index)
|
|
3278
|
+
|
|
3279
|
+
pred = np.asarray(pred)
|
|
3280
|
+
if pred.ndim != 1:
|
|
3281
|
+
if pred.shape[1] == 1:
|
|
3282
|
+
pred = pred.ravel()
|
|
3283
|
+
else:
|
|
3284
|
+
raise ValueError(
|
|
3285
|
+
f"Model predictions must be 1-D, got shape {pred.shape}"
|
|
3286
|
+
)
|
|
3287
|
+
return pd.Series(pred, index=data.index)
|
|
3288
|
+
|
|
3289
|
+
|
|
3290
|
+
def added_plot(
|
|
3291
|
+
formula: str,
|
|
3292
|
+
data: pd.DataFrame,
|
|
3293
|
+
lm_formula: Optional[str] = None,
|
|
3294
|
+
method: str = "loess",
|
|
3295
|
+
x: Optional[Union[str, int]] = None,
|
|
3296
|
+
offset: bool = True,
|
|
3297
|
+
**kwargs,
|
|
3298
|
+
):
|
|
3299
|
+
"""Create an added variable plot (R-flexplot ``added.plot()`` parity).
|
|
3300
|
+
|
|
3301
|
+
Residualizes the outcome on a conditioning model, adds the mean of the
|
|
3302
|
+
outcome back to the residuals (R's "maintain interpretation" step), and
|
|
3303
|
+
plots the chosen display variable against those residuals.
|
|
3304
|
+
|
|
3305
|
+
R-flexplot semantics (v0.8.0+; supersedes the v0.6.x behavior which
|
|
3306
|
+
plotted the first variable against doubly-residualized data):
|
|
3307
|
+
|
|
3308
|
+
- ``formula``: ``y ~ var1 + var2 + ...``.
|
|
3309
|
+
- Default display variable (``x=None``) is the **last** variable on the
|
|
3310
|
+
RHS of ``formula`` (R's default): ``y ~ x + z`` residualizes ``y`` on
|
|
3311
|
+
``x`` and plots ``z``.
|
|
3312
|
+
- ``lm_formula`` (optional): the fitted model used to residualize ``y``
|
|
3313
|
+
(e.g. ``"weight.loss ~ health * muscle.gain"``). Defaults to the
|
|
3314
|
+
remaining formula variables (all but the display variable).
|
|
3315
|
+
- ``x`` (optional): which variable to display. Either the column name
|
|
3316
|
+
or its 1-based position in ``formula``'s predictor list (R uses
|
|
3317
|
+
``x=2`` for the second).
|
|
3318
|
+
- ``offset`` (default ``True``): add the mean of ``y`` back onto the
|
|
3319
|
+
residuals so the y-axis keeps the outcome's scale (R does this).
|
|
3320
|
+
Pass ``False`` for raw centered residuals.
|
|
3321
|
+
- ``method``: smoother for the fitted line (default ``"loess"``,
|
|
3322
|
+
matching R).
|
|
3323
|
+
"""
|
|
3324
|
+
variables = parse_flexplot_formula(formula)
|
|
3325
|
+
_validate_data_for_plot(formula, data, variables, require_numeric_x=True)
|
|
3326
|
+
|
|
3327
|
+
y_var = variables["y"]
|
|
3328
|
+
all_x = [v for v in variables["all_x"] if ":" not in v] # atoms only
|
|
3329
|
+
if not all_x:
|
|
3330
|
+
# Degenerate: only an interaction term (or nothing) — fall back.
|
|
3331
|
+
return flexplot(formula, data, **kwargs)
|
|
3332
|
+
|
|
3333
|
+
# Resolve the display variable (R default: last on the RHS).
|
|
3334
|
+
if x is None:
|
|
3335
|
+
x_var = all_x[-1]
|
|
3336
|
+
elif isinstance(x, int):
|
|
3337
|
+
if not (1 <= x <= len(all_x)):
|
|
3338
|
+
raise ValueError(
|
|
3339
|
+
f"x={x} is out of range; formula has {len(all_x)} predictors."
|
|
3340
|
+
)
|
|
3341
|
+
x_var = all_x[x - 1]
|
|
3342
|
+
elif isinstance(x, str):
|
|
3343
|
+
if x not in all_x:
|
|
3344
|
+
raise ValueError(
|
|
3345
|
+
f"x={x!r} not found among formula predictors {all_x}."
|
|
3346
|
+
)
|
|
3347
|
+
x_var = x
|
|
3348
|
+
else:
|
|
3349
|
+
raise TypeError(f"x must be a str, int, or None; got {type(x).__name__}.")
|
|
3350
|
+
|
|
3351
|
+
# Conditioning variables: from lm_formula if given, else all formula
|
|
3352
|
+
# predictors except the display variable (R: "the fitted model that is
|
|
3353
|
+
# then residualized"). Handle interaction-expanded atoms.
|
|
3354
|
+
if lm_formula is not None:
|
|
3355
|
+
if not isinstance(lm_formula, str):
|
|
3356
|
+
raise TypeError(
|
|
3357
|
+
f"lm_formula must be a string; got {type(lm_formula).__name__}."
|
|
3358
|
+
)
|
|
3359
|
+
lm_variables = parse_flexplot_formula(
|
|
3360
|
+
lm_formula if "~" in lm_formula else f"{y_var} ~ {lm_formula}"
|
|
3361
|
+
)
|
|
3362
|
+
if lm_variables["y"] != y_var:
|
|
3363
|
+
raise ValueError(
|
|
3364
|
+
f"lm_formula {lm_formula!r} must share the outcome {y_var!r}; "
|
|
3365
|
+
f"it uses {lm_variables['y']!r}."
|
|
3366
|
+
)
|
|
3367
|
+
condition_vars = [v for v in lm_variables["all_x"] if ":" not in v]
|
|
3368
|
+
if x_var in condition_vars:
|
|
3369
|
+
condition_vars.remove(x_var)
|
|
3370
|
+
else:
|
|
3371
|
+
condition_vars = [v for v in all_x if v != x_var]
|
|
3372
|
+
if not condition_vars:
|
|
3373
|
+
# Only one predictor: nothing to condition on; plain flexplot.
|
|
3374
|
+
return flexplot(formula, data, **kwargs)
|
|
3375
|
+
|
|
3376
|
+
# Residualize y on the conditioning variables.
|
|
3377
|
+
clean_cond = [re.sub(r"\W", "_", v) for v in condition_vars]
|
|
3378
|
+
mapping = {orig: clean for orig, clean in zip(condition_vars, clean_cond)}
|
|
3379
|
+
needed = list(dict.fromkeys(condition_vars + [x_var, y_var]))
|
|
3380
|
+
fit_df = data[needed].dropna().copy()
|
|
3381
|
+
for orig, clean in mapping.items():
|
|
3382
|
+
if orig != clean:
|
|
3383
|
+
fit_df[clean] = fit_df.pop(orig)
|
|
3384
|
+
y_res_model = OLS.from_formula(
|
|
3385
|
+
f"{y_var} ~ {' + '.join(clean_cond)}", data=fit_df
|
|
3386
|
+
).fit()
|
|
3387
|
+
y_residuals = y_res_model.resid + y_res_model.model.endog.mean() if offset \
|
|
3388
|
+
else y_res_model.resid
|
|
3389
|
+
|
|
3390
|
+
plot_df = pd.DataFrame({
|
|
3391
|
+
x_var: fit_df[x_var].to_numpy(),
|
|
3392
|
+
f"{y_var}|cond": np.asarray(y_residuals),
|
|
3393
|
+
})
|
|
3394
|
+
|
|
3395
|
+
aes_kwargs = {"x": x_var, "y": f"{y_var}|cond"}
|
|
3396
|
+
p = ggplot(plot_df, aes(**aes_kwargs))
|
|
3397
|
+
p += geom_point(alpha=0.5)
|
|
3398
|
+
# Reuse the numeric-smooth machinery, honoring method / uncertainty /
|
|
3399
|
+
# level / bands kwargs when provided.
|
|
3400
|
+
kwargs_unc = kwargs.pop("uncertainty", "ci")
|
|
3401
|
+
kwargs_level = kwargs.pop("level", 0.95)
|
|
3402
|
+
kwargs_bands = kwargs.pop("bands", None)
|
|
3403
|
+
p = _add_numeric_smooth(
|
|
3404
|
+
p, plot_df, x_var, f"{y_var}|cond",
|
|
3405
|
+
method if method in _VALID_FLEXPLOT_METHODS else ("loess" if method == "auto" else method),
|
|
3406
|
+
kwargs_unc, kwargs_level, kwargs_bands,
|
|
3407
|
+
)
|
|
3408
|
+
p += labs(x=x_var, y=f"{y_var} | conditional", title="Added Variable Plot")
|
|
3409
|
+
p += theme_bw()
|
|
3410
|
+
return p
|