py-flexplot 0.8.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pyflexplot/core.py ADDED
@@ -0,0 +1,3410 @@
1
+ import ast
2
+ import re
3
+ import warnings
4
+
5
+ import pandas as pd
6
+ import numpy as np
7
+ from typing import Any, List, Optional, Union, cast
8
+ from plotnine import (
9
+ ggplot,
10
+ aes,
11
+ geom_histogram,
12
+ geom_point,
13
+ geom_smooth,
14
+ geom_jitter,
15
+ geom_line,
16
+ geom_ribbon,
17
+ geom_hline,
18
+ geom_boxplot,
19
+ geom_violin,
20
+ geom_bar,
21
+ geom_density,
22
+ stat_summary,
23
+ stat_qq,
24
+ stat_qq_line,
25
+ facet_wrap,
26
+ facet_grid,
27
+ scale_color_identity,
28
+ scale_color_manual,
29
+ labs,
30
+ theme_bw,
31
+ theme,
32
+ element_blank,
33
+ )
34
+ import statsmodels.api as sm
35
+ import statsmodels.formula.api as smf
36
+ from statsmodels.regression.linear_model import OLS
37
+ from statsmodels.nonparametric.smoothers_lowess import lowess
38
+
39
+ from .uncertainty import (
40
+ validate_uncertainty_params,
41
+ compute_bootstrap_ci,
42
+ )
43
+
44
+
45
+ def parse_flexplot_formula(formula: str):
46
+ """
47
+ Parses a flexplot formula of the form:
48
+ outcome ~ predictor1 + predictor2 | given1 + given2
49
+
50
+ Validates the formula syntax, strips whitespace from tokens, rejects empty
51
+ outcome/predictor, handles the intercept-only form ``y ~ 1`` explicitly,
52
+ and allows at most one ``|``.
53
+ """
54
+ if not isinstance(formula, str):
55
+ raise TypeError(f"formula must be a string, got {type(formula).__name__}")
56
+
57
+ if formula.count("~") != 1:
58
+ raise ValueError(
59
+ f"Formula must contain exactly one '~': {formula!r}"
60
+ )
61
+
62
+ if formula.count("|") > 1:
63
+ raise ValueError(
64
+ f"Formula may contain at most one '|': {formula!r}"
65
+ )
66
+
67
+ if "|" in formula:
68
+ main_part, given_part = formula.split("|", 1)
69
+ else:
70
+ main_part = formula
71
+ given_part = None
72
+
73
+ main_part = main_part.strip()
74
+ given_part = given_part.strip() if given_part is not None else None
75
+
76
+ y_name, sep, x_formula = main_part.partition("~")
77
+ y_name = y_name.strip()
78
+ x_formula = x_formula.strip()
79
+
80
+ if not y_name:
81
+ raise ValueError(f"Formula must have a non-empty outcome: {formula!r}")
82
+ if not x_formula:
83
+ raise ValueError(
84
+ f"Formula must have predictors after '~' (use 'y ~ 1' for intercept-only): {formula!r}"
85
+ )
86
+
87
+ # Intercept-only formula: y ~ 1
88
+ if x_formula == "1":
89
+ return {
90
+ "y": y_name,
91
+ "x": None,
92
+ "color": None,
93
+ "given": [g.strip() for g in given_part.split("+")] if given_part else [],
94
+ "all_x": [],
95
+ "intercept_only": True,
96
+ "has_interaction": False,
97
+ }
98
+
99
+ # Detect interaction operators (``*`` or ``:``) anywhere in the
100
+ # right-hand side. The current fit is still additive; we set a flag
101
+ # so ``flexplot()`` can warn the user. ``*`` is expanded to its R-style
102
+ # constituent terms (``a*b`` → ``a + b + a:b``) so column lookup works.
103
+ has_interaction = bool(_INTERACTION_OP.search(x_formula))
104
+ if has_interaction:
105
+ expanded_x_formula = _expand_r_formula(x_formula)
106
+ else:
107
+ expanded_x_formula = x_formula
108
+
109
+ x_parts = [p.strip() for p in expanded_x_formula.split("+")]
110
+ x_parts = [p for p in x_parts if p]
111
+
112
+ if not x_parts:
113
+ raise ValueError(
114
+ f"Formula must have at least one predictor after '~': {formula!r}"
115
+ )
116
+
117
+ # ``x_name`` is the first atom of the first term (so ``x:z`` → ``x``);
118
+ # ``color_name`` is the first atom of the second term if present.
119
+ x_name = _first_atom(x_parts[0])
120
+ color_name = _first_atom(x_parts[1]) if len(x_parts) > 1 else None
121
+
122
+ given_names = [g.strip() for g in given_part.split("+")] if given_part else []
123
+ given_names = [g for g in given_names if g]
124
+
125
+ return {
126
+ "y": y_name,
127
+ "x": x_name,
128
+ "color": color_name,
129
+ "given": given_names,
130
+ "all_x": x_parts,
131
+ "intercept_only": False,
132
+ "has_interaction": has_interaction,
133
+ }
134
+
135
+
136
+ def _validate_data_for_plot(
137
+ formula: str,
138
+ data: pd.DataFrame,
139
+ variables: dict,
140
+ require_numeric_x: bool = False,
141
+ intercept_only: bool = False,
142
+ ):
143
+ """Shared validation for flexplot and added_plot."""
144
+ if not isinstance(data, pd.DataFrame):
145
+ raise TypeError(
146
+ f"data must be a pandas DataFrame, got {type(data).__name__}"
147
+ )
148
+ if data.empty:
149
+ raise ValueError(
150
+ f"data must be a non-empty DataFrame for formula {formula!r}"
151
+ )
152
+
153
+ y = variables["y"]
154
+ x = variables["x"]
155
+ given = variables.get("given", [])
156
+ color = variables.get("color")
157
+
158
+ required = {y}
159
+ if x is not None:
160
+ required.add(x)
161
+ if color is not None:
162
+ required.add(color)
163
+ for g in given:
164
+ required.add(g)
165
+
166
+ missing = sorted(required - set(data.columns))
167
+ if missing:
168
+ raise ValueError(
169
+ f"Formula {formula!r} references missing columns in data: {missing}"
170
+ )
171
+
172
+ # Validate outcome column y is numeric (or numeric-convertible).
173
+ # For intercept-only formulas, R-flexplot supports univariate plots of
174
+ # categorical outcomes (bar charts), so we relax the numeric requirement.
175
+ if (
176
+ y is not None
177
+ and not intercept_only
178
+ and not pd.api.types.is_numeric_dtype(data[y])
179
+ ):
180
+ try:
181
+ pd.to_numeric(data[y].dropna())
182
+ except (ValueError, TypeError):
183
+ raise ValueError(
184
+ f"Column {y!r} must be numeric for formula {formula!r}, "
185
+ f"got dtype {data[y].dtype}"
186
+ ) from None
187
+
188
+ if require_numeric_x and x is not None and not pd.api.types.is_numeric_dtype(data[x]):
189
+ raise ValueError(
190
+ f"Column {x!r} must be numeric for formula {formula!r}, "
191
+ f"got dtype {data[x].dtype}"
192
+ )
193
+
194
+
195
+ # Threshold below which R-flexplot converts numeric predictors to ordered
196
+ # categorical factors. Matches R's ``convert_if_less_than_five`` (numeric
197
+ # with <5 unique values -> ordered factor).
198
+ _LOW_CARDINALITY_THRESHOLD = 5
199
+
200
+
201
+ def _is_low_cardinality_numeric(series: pd.Series) -> bool:
202
+ """Return True if ``series`` is numeric with fewer than 5 unique non-null values.
203
+
204
+ Mirrors R-flexplot's ``convert_if_less_than_five``: numeric axis / color /
205
+ given variables with 2-4 unique values should be treated as ordered
206
+ categorical factors, both for plotting and for the smoother fit. Series
207
+ that are already non-numeric, or numeric with 5+ unique values, return
208
+ False (no conversion needed).
209
+ """
210
+ if not pd.api.types.is_numeric_dtype(series):
211
+ return False
212
+ n_unique = series.dropna().nunique()
213
+ return 0 < n_unique < _LOW_CARDINALITY_THRESHOLD
214
+
215
+
216
+ def _is_discrete(series: pd.Series) -> bool:
217
+ """
218
+ Returns True if the series is non-numeric (string, object, categorical, bool)
219
+ or is numeric with 10 or fewer unique non-null values.
220
+
221
+ For R-parity low-cardinality conversion (numeric with <5 unique values
222
+ becomes ordered categorical), see ``_is_low_cardinality_numeric`` /
223
+ ``_convert_low_cardinality_to_categorical``.
224
+ """
225
+ if not pd.api.types.is_numeric_dtype(series):
226
+ return True
227
+ return series.dropna().nunique() <= 10
228
+
229
+
230
+ def _convert_low_cardinality_to_categorical(
231
+ data: pd.DataFrame,
232
+ variables: list,
233
+ ):
234
+ """Convert numeric predictors with <5 unique values to string (categorical).
235
+
236
+ R-flexplot's ``convert_if_less_than_five`` turns these into ordered
237
+ factors so the discrete-x branch (geom_jitter + dispersion marker)
238
+ applies instead of the numeric-x smoother. In Python we convert to
239
+ ``str`` so plotnine treats the column as discrete; the actual fitted
240
+ model still uses the (now string) values via statsmodels' C() wrapper
241
+ when needed.
242
+
243
+ Returns a new DataFrame (the input is not mutated). Variables that
244
+ are missing from ``data`` are silently skipped (callers should have
245
+ validated columns earlier).
246
+ """
247
+ out = data.copy()
248
+ for var in variables:
249
+ if var is None or var not in out.columns:
250
+ continue
251
+ if _is_low_cardinality_numeric(out[var]):
252
+ out[var] = out[var].astype(str)
253
+ return out
254
+
255
+
256
+ def _validate_binning_params(
257
+ bins,
258
+ labels,
259
+ breaks,
260
+ x_series: pd.Series,
261
+ ):
262
+ """Validate bins / labels / breaks arguments for numeric-x discretization.
263
+
264
+ Rules:
265
+ - ``bins``: positive int >= 2 (1 bin is meaningless).
266
+ - ``breaks``: list of floats, length >= 2, strictly monotonically
267
+ increasing.
268
+ - ``labels``: list of strings. When given with ``breaks``, len must be
269
+ len(breaks) - 1. When given with ``bins`` alone, len must equal
270
+ ``bins``.
271
+ - ``bins`` and ``breaks`` are mutually exclusive (breaks wins).
272
+ - All binning params are silently ignored when x is already discrete
273
+ or non-numeric (caller checks first).
274
+ """
275
+ if bins is None and breaks is None and labels is None:
276
+ return
277
+ if bins is not None:
278
+ if not isinstance(bins, int) or isinstance(bins, bool):
279
+ raise TypeError(
280
+ f"bins must be an int >= 2; got {type(bins).__name__} ({bins!r})."
281
+ )
282
+ if bins < 2:
283
+ raise ValueError(f"bins must be >= 2; got {bins}.")
284
+ if breaks is not None:
285
+ if not isinstance(breaks, (list, tuple)):
286
+ raise TypeError(
287
+ f"breaks must be a list/tuple of floats; got {type(breaks).__name__}."
288
+ )
289
+ if len(breaks) < 2:
290
+ raise ValueError(
291
+ f"breaks must have >= 2 cut points; got {len(breaks)}."
292
+ )
293
+ # Coerce to float and check monotonicity.
294
+ breaks_f = [float(b) for b in breaks]
295
+ for i in range(1, len(breaks_f)):
296
+ if breaks_f[i] <= breaks_f[i - 1]:
297
+ raise ValueError(
298
+ f"breaks must be strictly monotonically increasing; "
299
+ f"got {breaks_f!r}."
300
+ )
301
+ if labels is not None:
302
+ if not isinstance(labels, (list, tuple)):
303
+ raise TypeError(
304
+ f"labels must be a list/tuple of strings; got {type(labels).__name__}."
305
+ )
306
+ if any(not isinstance(lbl, str) for lbl in labels):
307
+ raise TypeError("labels must all be strings.")
308
+ if breaks is not None:
309
+ if len(labels) != len(breaks) - 1:
310
+ raise ValueError(
311
+ f"labels length ({len(labels)}) must equal "
312
+ f"len(breaks) - 1 ({len(breaks) - 1})."
313
+ )
314
+ elif bins is not None:
315
+ if len(labels) != bins:
316
+ raise ValueError(
317
+ f"labels length ({len(labels)}) must equal bins ({bins})."
318
+ )
319
+ if bins is not None and breaks is not None:
320
+ warnings.warn(
321
+ "Both bins and breaks were provided; breaks takes precedence.",
322
+ UserWarning,
323
+ stacklevel=3,
324
+ )
325
+
326
+
327
+ def _maybe_bin_numeric_x(
328
+ data: pd.DataFrame,
329
+ x: str,
330
+ bins=None,
331
+ labels=None,
332
+ breaks=None,
333
+ ):
334
+ """Discretize a numeric x column into bins/breaks.
335
+
336
+ Returns (dataframe, was_binned: bool). If neither bins nor breaks is
337
+ given, returns (data.copy(), False) without modifying x.
338
+
339
+ Uses pd.cut() for both equal-width (bins) and explicit-cut (breaks)
340
+ paths. NaN handling: rows with NaN x are dropped from the binning but
341
+ preserved in the returned dataframe with NaN x (plotnine will skip them).
342
+ """
343
+ if bins is None and breaks is None:
344
+ return data.copy(), False
345
+
346
+ x_arr = data[x].to_numpy()
347
+ if breaks is not None:
348
+ cuts = list(breaks)
349
+ else:
350
+ # Equal-width bins between min and max (inclusive on the lower end).
351
+ x_min = float(np.nanmin(x_arr))
352
+ x_max = float(np.nanmax(x_arr))
353
+ cuts = np.linspace(x_min, x_max, num=int(bins) + 1).tolist()
354
+
355
+ # Ensure endpoints are captured even if the data doesn't hit them.
356
+ # pd.cut's include_lowest=True makes the leftmost bin closed on both ends.
357
+ binned = pd.cut(
358
+ data[x],
359
+ bins=cuts,
360
+ labels=labels,
361
+ include_lowest=True,
362
+ )
363
+
364
+ out = data.copy()
365
+ # Convert to string so plotnine treats it as discrete levels.
366
+ out[x] = binned.astype(str)
367
+ return out, True
368
+
369
+
370
+ _VALID_SPREAD = frozenset({None, "stdev", "range", "iqr", "no", "ci", "quartiles", "sterr"})
371
+
372
+
373
+ def _add_discrete_summary(p, spread: Optional[str]):
374
+ """Add the dispersion marker layer for the discrete-x branch.
375
+
376
+ Mirrors R-flexplot's ``spread`` argument:
377
+ - None / "quartiles": median +/- Q1/Q3 IQR (R's default for discrete x).
378
+ - "iqr": alias for "quartiles".
379
+ - "ci": bootstrap CI on the mean (plotnine's stat_summary with
380
+ ``fun_data='mean_cl_boot'``).
381
+ - "sterr": mean +/- 1.96 * standard error of the mean.
382
+ - "stdev": mean +/- 1 SD as a crossbar (pointrange with computed limits).
383
+ - "range": min-max range as a wider crossbar.
384
+ - "no": no summary layer at all.
385
+ """
386
+ if spread not in _VALID_SPREAD:
387
+ raise ValueError(
388
+ f"spread must be one of {sorted(s for s in _VALID_SPREAD if s)}; "
389
+ f"got {spread!r}."
390
+ )
391
+
392
+ # R-token aliases: "quartiles" == "iqr". Default to "iqr" to
393
+ # match R-flexplot's discrete-x default; legacy Python callers can
394
+ # request "ci" explicitly.
395
+ if spread is None or spread == "quartiles":
396
+ spread = "iqr"
397
+
398
+ if spread == "no":
399
+ return p
400
+
401
+ if spread == "ci":
402
+ p += stat_summary(fun_data="mean_cl_boot", color="red", size=1)
403
+ return p
404
+
405
+ if spread == "sterr":
406
+ # Standard error of the mean: sd / sqrt(n). R-flexplot uses the
407
+ # same formula with a historical n-1 denominator; we use the
408
+ # conventional sample-size denominator for consistency with
409
+ # statsmodels / scipy.
410
+ fun = _make_spread_fn(
411
+ np.mean,
412
+ lambda x: (
413
+ np.mean(x) - 1.96 * (np.std(x, ddof=1) / np.sqrt(len(x))),
414
+ np.mean(x) + 1.96 * (np.std(x, ddof=1) / np.sqrt(len(x))),
415
+ ),
416
+ )
417
+ p += stat_summary(fun_data=fun, geom="pointrange", color="red", size=0.5)
418
+ return p
419
+
420
+ # stdev / range / iqr: use a precomputed summary dataframe + pointrange.
421
+ # stat_summary can't easily express "by group" summaries that return a
422
+ # single (y, ymin, ymax) per x level, so we build it manually.
423
+ # Pull the aes from the existing plot: x_var is the discrete-x column.
424
+ # We don't know the column names here without the caller passing them,
425
+ # so we use the plot's already-attached data + aes.
426
+ #
427
+ # IMPORTANT: callers should prefer plot-level data extraction. For
428
+ # simplicity we use a fallback: invoke stat_summary with a custom
429
+ # fun_data that yields (ymin, y) by computing per-level quantiles.
430
+ # The summary fn must return a DataFrame with columns 'y', 'ymin', 'ymax'
431
+ # and an 'x' level column.
432
+ if spread == "stdev":
433
+ fun = _make_spread_fn(np.mean, lambda x: np.std(x, ddof=1))
434
+ elif spread == "range":
435
+ fun = _make_spread_fn(np.mean, lambda x: (np.min(x), np.max(x)))
436
+ elif spread == "iqr":
437
+ fun = _make_spread_fn(np.median, lambda x: (np.percentile(x, 25), np.percentile(x, 75)))
438
+ else: # pragma: no cover — guarded by validator
439
+ return p
440
+
441
+ p += stat_summary(fun_data=fun, geom="pointrange", color="red", size=0.5)
442
+ return p
443
+
444
+
445
+ def _plot_univariate(
446
+ data: pd.DataFrame,
447
+ outcome: str,
448
+ plot_type: Optional[str] = None,
449
+ bins: Optional[int] = None,
450
+ ):
451
+ """Build an intercept-only / univariate distribution plot.
452
+
453
+ Mirrors ``r-flexplot/R/flexplot_helper.R::flexplot_histogram`` plus the
454
+ bivariate ``plot.type`` variants. ``outcome`` is the variable being
455
+ visualized (usually the formula's ``y``).
456
+
457
+ Parameters
458
+ ----------
459
+ data : pd.DataFrame
460
+ Plotting data frame. May already be a subsample when ``sample=`` is
461
+ used, but the caller decides that before invoking this helper.
462
+ outcome : str
463
+ Column name of the variable to plot.
464
+ plot_type : {None, "histogram", "qq", "density", "boxplot", "violin"}, optional
465
+ Univariate geom override. ``None`` defaults to a histogram.
466
+ bins : int, optional
467
+ Number of histogram bins. Ignored for non-histogram plot types.
468
+
469
+ Returns
470
+ -------
471
+ plotnine.ggplot
472
+ A complete univariate plot with ``theme_bw`` and appropriate axis
473
+ labeling.
474
+ """
475
+ plot_type = plot_type or "histogram"
476
+ is_numeric = pd.api.types.is_numeric_dtype(data[outcome])
477
+
478
+ # Categorical outcome: R draws a bar chart regardless of plot_type.
479
+ if not is_numeric:
480
+ p = ggplot(data, aes(x=outcome)) + geom_bar()
481
+ p += labs(x=outcome, title=f"Distribution of {outcome}")
482
+ p += theme_bw()
483
+ return p
484
+
485
+ if plot_type == "qq":
486
+ p = ggplot(data, aes(sample=outcome))
487
+ p += stat_qq()
488
+ p += stat_qq_line()
489
+ p += labs(title=f"QQ plot of {outcome}")
490
+ p += theme_bw()
491
+ return p
492
+
493
+ if plot_type == "density":
494
+ p = ggplot(data, aes(x=outcome)) + geom_density()
495
+ p += labs(x=outcome, title=f"Density of {outcome}")
496
+ p += theme_bw()
497
+ return p
498
+
499
+ if plot_type in {"boxplot", "violin"}:
500
+ geom = geom_boxplot() if plot_type == "boxplot" else geom_violin()
501
+ p = ggplot(data, aes(y=outcome)) + geom
502
+ p += labs(y=outcome, title=f"Distribution of {outcome}")
503
+ p += theme_bw()
504
+ # Hide the redundant x-axis markings (the natural analogue of
505
+ # R's coord_flip + blank x-axis for a univariate boxplot).
506
+ p += theme(
507
+ axis_title_x=element_blank(),
508
+ axis_text_x=element_blank(),
509
+ axis_ticks_major_x=element_blank(),
510
+ )
511
+ return p
512
+
513
+ # Default / explicit histogram
514
+ n_bins = bins if bins is not None else 30
515
+ p = ggplot(data, aes(x=outcome)) + geom_histogram(
516
+ bins=n_bins, fill="lightgray", color="black"
517
+ )
518
+ p += labs(x=outcome, title=f"Distribution of {outcome}")
519
+ p += theme_bw()
520
+ return p
521
+
522
+
523
+ def _plot_related(
524
+ data: pd.DataFrame,
525
+ diff_col: str,
526
+ spread: Optional[str],
527
+ plot_type: Optional[str],
528
+ jitter: Union[bool, tuple, List[float], None],
529
+ alpha: float,
530
+ raw_data: bool,
531
+ ):
532
+ """Build a related-samples / paired difference plot.
533
+
534
+ Mirrors ``r-flexplot/R/flexplot_helper.R::flexplot_related``. The input
535
+ ``data`` is expected to contain a single column ``diff_col`` of paired
536
+ difference scores.
537
+ """
538
+ p = ggplot(data, aes(x=1, y=diff_col)) + theme_bw()
539
+ p += geom_hline(yintercept=0, color="lightgray")
540
+ p += labs(y=diff_col, title=f"{diff_col}")
541
+ p += theme(
542
+ axis_title_x=element_blank(),
543
+ axis_text_x=element_blank(),
544
+ axis_ticks_major_x=element_blank(),
545
+ )
546
+
547
+ if plot_type in {"boxplot", "violin"}:
548
+ geom = geom_boxplot() if plot_type == "boxplot" else geom_violin()
549
+ p += geom
550
+ else:
551
+ # Default/errorbar path: show jittered points + a dispersion marker.
552
+ if raw_data:
553
+ if jitter is None:
554
+ jitter_xy = (0.05, 0.0)
555
+ elif isinstance(jitter, bool):
556
+ jitter_xy = (0.05, 0.0) if jitter else (0.0, 0.0)
557
+ else:
558
+ jitter_xy = (float(jitter[0]), float(jitter[1]) if len(jitter) > 1 else 0.0)
559
+ if jitter_xy[0] > 0:
560
+ p += geom_jitter(width=jitter_xy[0], height=jitter_xy[1], alpha=alpha)
561
+ else:
562
+ p += geom_point(alpha=alpha)
563
+ p = _add_discrete_summary(p, spread)
564
+
565
+ return p
566
+
567
+
568
+ def _make_spread_fn(center_fn, spread_fn):
569
+ """Build a plotnine fun_data-style callable for stat_summary.
570
+
571
+ Returns a function ``f(values: np.ndarray) -> pd.DataFrame`` with one row
572
+ containing columns ``y``, ``ymin``, ``ymax`` (plotnine's expected schema
573
+ for ``pointrange``). The center_fn is applied to compute ``y``; the
574
+ spread_fn is applied to compute (ymin, ymax).
575
+
576
+ Note: plotnine's stat_summary fun_data expects the x-level grouping to
577
+ be handled internally. We rely on the default ``fun_y=np.mean`` for the
578
+ point and our custom fun_data for the range. If the caller's spread_fn
579
+ returns a 2-tuple (lo, hi), we project those into ymin / ymax.
580
+ """
581
+ def _f(values):
582
+ center = center_fn(values)
583
+ spread = spread_fn(values)
584
+ if isinstance(spread, tuple) and len(spread) == 2:
585
+ lo, hi = spread
586
+ else: # pragma: no cover — defensive
587
+ lo, hi = center - spread, center + spread
588
+ return pd.DataFrame({"y": [center], "ymin": [lo], "ymax": [hi]})
589
+ return _f
590
+
591
+
592
+ _VALID_FLEXPLOT_METHODS = frozenset(
593
+ {
594
+ "auto",
595
+ "lm",
596
+ "loess",
597
+ "quadratic",
598
+ "polynomial",
599
+ "cubic",
600
+ "logistic",
601
+ "rlm",
602
+ "poisson",
603
+ "Gamma",
604
+ # Mixed-effects extensions (v0.8.2+):
605
+ "mixedlm",
606
+ "lmer",
607
+ "glmer",
608
+ }
609
+ )
610
+
611
+ # Recognized methods for overlay entries. Includes a broader set than the
612
+ # primary ``method`` parameter because plotnine/statsmodels supports more
613
+ # smoothers for overlay use.
614
+ _VALID_OVERLAY_METHODS = frozenset({"lm", "loess", "lowess", "glm", "rlm", "ols", "wls", "gls", "mavg"})
615
+
616
+ # Recognized plot_type overrides. Histogram/QQ/density/violin are used
617
+ # primarily for intercept-only (univariate) plots but are accepted anywhere
618
+ # the data type permits them.
619
+ _VALID_PLOT_TYPES = frozenset(
620
+ {"scatter", "line", "boxplot", "bar", "histogram", "qq", "density", "violin"}
621
+ )
622
+
623
+ # Default color cycle for overlay entries (distinct from the primary
624
+ # ``"blue"`` so the primary line is always visually identifiable).
625
+ _OVERLAY_COLOR_CYCLE = ("#e74c3c", "#2ecc71", "#9b59b6", "#f39c12", "#1abc9c")
626
+
627
+ # Interaction-operator detection. The presence of ``*`` or ``:`` in the
628
+ # right-hand side of a formula signals that the user wants interaction terms.
629
+ # The parser accepts these for forward-compatibility with v0.7.0 (real
630
+ # interaction-aware fitting), but the default fit in v0.6.x is still
631
+ # additive — a UserWarning is emitted to make this explicit.
632
+ _INTERACTION_OP = re.compile(r"(?<!\*)\*(?!\*)|:")
633
+
634
+
635
+ def _split_formula_terms(text: str, sep: str = "+") -> List[str]:
636
+ """Split ``text`` on ``sep`` only outside parentheses.
637
+
638
+ Used to split the RHS of a flexplot formula into additive terms without
639
+ breaking apart function calls such as ``I(x ** 2 + 1)``.
640
+ """
641
+ depth = 0
642
+ current: List[str] = []
643
+ terms: List[str] = []
644
+ for ch in text:
645
+ if ch == "(":
646
+ depth += 1
647
+ elif ch == ")":
648
+ depth -= 1
649
+ elif ch == sep and depth == 0:
650
+ term = "".join(current).strip()
651
+ if term:
652
+ terms.append(term)
653
+ current = []
654
+ continue
655
+ current.append(ch)
656
+ term = "".join(current).strip()
657
+ if term:
658
+ terms.append(term)
659
+ return terms
660
+
661
+
662
+ def _expand_r_formula(text: str) -> str:
663
+ """Expand R-style ``a*b`` to ``a + b + a:b``.
664
+
665
+ Repeatedly applies the expansion until no ``*`` remains (handles
666
+ multi-way interactions like ``a*b*c``). ``:`` terms are left as-is;
667
+ downstream code can choose how to handle them.
668
+ """
669
+ if "*" not in text:
670
+ return text
671
+ pattern = re.compile(r"(\b\w+)\s*\*\s*(\w+)")
672
+ while True:
673
+ new_text = pattern.sub(r"\1 + \2 + \1:\2", text)
674
+ if new_text == text:
675
+ return text
676
+ text = new_text
677
+
678
+
679
+ def _first_atom(term: str) -> str:
680
+ """Return the first atom of a possibly-interacted term.
681
+
682
+ ``x:z`` → ``x``; ``x`` → ``x``. Used to extract the column name when
683
+ the parser encounters interaction terms.
684
+ """
685
+ return term.split(":", 1)[0].strip()
686
+
687
+
688
+ def _normalize_overlay(overlay):
689
+ """Validate and normalize the ``overlay`` parameter into a list of dicts.
690
+
691
+ Each returned dict has at least:
692
+ - ``method``: str (required)
693
+ - ``color``: str (default: next color from cycle)
694
+ - ``label``: str (default: method name)
695
+ - ``uncertainty``: {None, "ci", "prediction", "bootstrap"}, default "ci"
696
+ - ``level``: float in (0, 1), default 0.95
697
+
698
+ Raises ``ValueError`` if any entry is malformed.
699
+ """
700
+ if overlay is None:
701
+ return []
702
+ if not isinstance(overlay, (list, tuple)):
703
+ raise ValueError(
704
+ f"overlay must be a list or tuple; got {type(overlay).__name__}."
705
+ )
706
+ if not overlay:
707
+ return []
708
+
709
+ normalized = []
710
+ for i, entry in enumerate(overlay):
711
+ if isinstance(entry, str):
712
+ spec = {"method": entry}
713
+ elif isinstance(entry, dict):
714
+ spec = dict(entry)
715
+ else:
716
+ raise ValueError(
717
+ f"overlay entry {i} must be a str or dict; "
718
+ f"got {type(entry).__name__}."
719
+ )
720
+ if "method" not in spec:
721
+ raise ValueError(
722
+ f"overlay entry {i} is missing required key 'method': {entry!r}."
723
+ )
724
+ if spec["method"] not in _VALID_OVERLAY_METHODS:
725
+ raise ValueError(
726
+ f"overlay entry {i}: method {spec['method']!r} is not a "
727
+ f"recognized method. Valid: {sorted(_VALID_OVERLAY_METHODS)}."
728
+ )
729
+ spec.setdefault("color", _OVERLAY_COLOR_CYCLE[i % len(_OVERLAY_COLOR_CYCLE)])
730
+ spec.setdefault("label", spec["method"])
731
+ spec.setdefault("uncertainty", "ci")
732
+ spec.setdefault("level", 0.95)
733
+ normalized.append(spec)
734
+ return normalized
735
+
736
+
737
+ # ---------------------------------------------------------------------------
738
+ # Formula-function detection / evaluation (R-flexplot parity, v0.8.x+).
739
+ # R's ``formula_functions`` looks for terms containing ``(`` (e.g. ``log(x)``,
740
+ # ``sqrt(x)``, ``poly(x, 2)``), applies the expression to ``data`` (via R's
741
+ # ``eval(parse(...))`` with the data as the evaluation environment), stores
742
+ # the result in a column named after the inner variable (e.g. ``x``), and
743
+ # rewrites the formula so downstream code sees ``y ~ x``.
744
+ #
745
+ # Python parity note: we use a SAFE whitelisted evaluator (numpy / pandas /
746
+ # math / statsmodels / patsy built-ins) rather than ``eval()`` on arbitrary
747
+ # strings, so a user-supplied formula can only invoke a closed set of
748
+ # known-safe functions. Unknown functions raise ``ValueError``.
749
+ # ---------------------------------------------------------------------------
750
+
751
+
752
+ # Whitelisted functions available inside formula terms. Keys are the names
753
+ # users may write (lowercase, since formula evaluation is case-insensitive
754
+ # for Python identifiers matched against this map); values are the callable.
755
+ _FORMULA_FUNCS = {
756
+ # numpy ufuncs / reductions
757
+ "log": np.log,
758
+ "log2": np.log2,
759
+ "log10": np.log10,
760
+ "log1p": np.log1p,
761
+ "exp": np.exp,
762
+ "exp2": np.exp2,
763
+ "expm1": np.expm1,
764
+ "sqrt": np.sqrt,
765
+ "abs": np.abs,
766
+ "abs_": np.abs, # alias to handle ``abs()`` when ``abs`` shadows builtin
767
+ "sign": np.sign,
768
+ "round": np.round,
769
+ "floor": np.floor,
770
+ "ceil": np.ceil,
771
+ "sin": np.sin,
772
+ "cos": np.cos,
773
+ "tan": np.tan,
774
+ "asin": np.arcsin,
775
+ "acos": np.arccos,
776
+ "atan": np.arctan,
777
+ "sinh": np.sinh,
778
+ "cosh": np.cosh,
779
+ "tanh": np.tanh,
780
+ # math module (scalar → scalar; will be wrapped to vectorize below)
781
+ "log_m": __import__("math").log,
782
+ "exp_m": __import__("math").exp,
783
+ "sqrt_m": __import__("math").sqrt,
784
+ # I(): identity (no-op; the ``I(x**2)`` R idiom)
785
+ "I": (lambda x: x),
786
+ # poly(): raw polynomial of given degree; default degree=2 (matches R's
787
+ # default ``poly(x, 2, raw=TRUE)``). Returns a numpy array with columns
788
+ # ``[x, x^2, ..., x^degree]``; we use the highest-degree column as the
789
+ # value (R returns the full matrix but ``formula_functions`` only stores
790
+ # it under the inner-variable name; we keep the highest non-linear
791
+ # term so downstream plots/smoothers see a single transformed column).
792
+ "poly": None, # filled in by _apply_formula_function (needs degree kwarg)
793
+ }
794
+
795
+
796
+ def _apply_formula_function(func_name: str, inner_expr: str, var_name: Optional[str],
797
+ data: pd.DataFrame, depth: int = 0):
798
+ """Apply a whitelisted function to a single inner expression.
799
+
800
+ ``inner_expr`` is the raw text inside the parentheses, e.g. ``"x"`` or
801
+ ``"x, 2"``. Returns a numpy array of values, ready to be stored in a
802
+ column. Raises ``ValueError`` for unknown functions or expressions
803
+ that reference missing / unknown columns.
804
+
805
+ Supports a single positional argument (the inner variable) plus
806
+ optional numeric constants (e.g. ``poly(x, 2)``). Inner expressions
807
+ like ``a + b`` are not supported (R's ``eval(parse(...))`` would allow
808
+ them, but we deliberately restrict to a single column reference so
809
+ there's no way for the formula string to reach other columns).
810
+ """
811
+ if depth > 3:
812
+ raise ValueError(
813
+ f"Nested formula functions beyond depth 3 are not supported; "
814
+ f"got {inner_expr!r} inside {func_name!r}."
815
+ )
816
+
817
+ parts = [p.strip() for p in inner_expr.split(",")]
818
+ if not parts or not parts[0]:
819
+ raise ValueError(
820
+ f"Empty inner expression for {func_name}(...): {inner_expr!r}"
821
+ )
822
+ inner_var = parts[0]
823
+ if inner_var not in data.columns:
824
+ raise ValueError(
825
+ f"Formula function {func_name}({inner_expr!r}) references "
826
+ f"missing column {inner_var!r}; available: {list(data.columns)}."
827
+ )
828
+ inner_arr = data[inner_var].to_numpy()
829
+
830
+ # poly(x, k): numpy polyfeatures [1, x, x^2, ..., x^k]. R uses raw
831
+ # (un-orthogonalized) polynomials by default; we follow suit for parity.
832
+ # Since we only store ONE column named after ``inner_var`` (R also
833
+ # stores the whole matrix under that name), we keep the highest-degree
834
+ # polynomial column — i.e. x^k for degree k. Users who want the full
835
+ # design matrix should construct it via method='polynomial' instead.
836
+ if func_name == "poly":
837
+ try:
838
+ degree = int(parts[1]) if len(parts) > 1 else 2
839
+ except (ValueError, TypeError):
840
+ raise ValueError(
841
+ f"poly() requires an integer degree; got {parts[1]!r}."
842
+ ) from None
843
+ if degree < 1:
844
+ raise ValueError(
845
+ f"poly() requires degree >= 1; got {degree}."
846
+ )
847
+ return np.asarray(inner_arr, dtype=float) ** degree
848
+
849
+ if func_name not in _FORMULA_FUNCS:
850
+ raise ValueError(
851
+ f"Formula function {func_name!r} is not supported; "
852
+ f"allowed names: {sorted(k for k in _FORMULA_FUNCS if k)}."
853
+ )
854
+ fn = _FORMULA_FUNCS[func_name]
855
+
856
+ # For numpy ufuncs, calling on a numpy array vectorizes correctly. For
857
+ # math.* scalar funcs (e.g. math.sqrt), wrap to vectorize. ``I()`` is
858
+ # a no-op passthrough; we handled it above via the identity lambda.
859
+ try:
860
+ result = fn(inner_arr)
861
+ except Exception as exc:
862
+ raise ValueError(
863
+ f"Failed to apply formula function {func_name}() to column "
864
+ f"{inner_var!r}: {exc}"
865
+ ) from exc
866
+ return np.asarray(result)
867
+
868
+
869
+ # Regex matching a term of the form ``funcName(inner[, args])``. Greedy
870
+ # only on the inner-var portion, so ``log(x)`` matches as ``log`` + ``x``.
871
+ _FORMULA_FUNC_RE = re.compile(
872
+ r"^\s*([A-Za-z_][A-Za-z_0-9]*)\s*\(\s*([^()]+)\s*\)\s*$"
873
+ )
874
+
875
+
876
+ def _apply_formula_functions(formula: str, data: pd.DataFrame):
877
+ """Detect and evaluate formula functions (R ``formula_functions`` parity).
878
+
879
+ Scans the right-hand side of ``formula`` for any term containing ``(``
880
+ (i.e. a function call). For each, applies the whitelisted function to
881
+ the referenced column(s), stores the result in a new column named after
882
+ the inner variable (overwriting it if present — R behavior), and
883
+ rewrites the formula so downstream code sees the simpler
884
+ ``inner_var`` name.
885
+
886
+ Returns
887
+ -------
888
+ (new_data, new_formula, transformed_terms)
889
+ ``new_data`` is the input DataFrame augmented with the new
890
+ transformed columns. ``new_formula`` is the rewritten formula
891
+ string. ``transformed_terms`` is a list of ``(term, func_name,
892
+ inner_var)`` tuples describing each transformation that was
893
+ applied (empty list when the formula has no functions).
894
+ """
895
+ if "|" not in formula:
896
+ main_part = formula
897
+ given_part = None
898
+ else:
899
+ main_part, given_part = formula.split("|", 1)
900
+
901
+ if "~" not in main_part:
902
+ # No predictor side: nothing to transform.
903
+ return data.copy(), formula, []
904
+
905
+ y_part, x_part = main_part.split("~", 1)
906
+ x_part = x_part.strip()
907
+ y_part = y_part.strip()
908
+
909
+ # Split the RHS into additive terms without breaking apart function
910
+ # calls such as ``I(x ** 2 + 1)``.
911
+ raw_terms = _split_formula_terms(x_part, "+")
912
+ if not raw_terms:
913
+ return data.copy(), formula, []
914
+
915
+ transformed = [] # list of (original_term, func_name, inner_var)
916
+ new_data = data.copy()
917
+ new_x_parts: List[str] = []
918
+
919
+ def _process_term(term: str, parts: List[str]) -> None:
920
+ """Apply formula-function transformation to one additive term.
921
+
922
+ Appends the rewritten term to ``parts`` and updates ``new_data`` /
923
+ ``transformed`` in the enclosing scope.
924
+ """
925
+ # If the term is an interaction (``a:b`` or ``a*b``) AND it's not a
926
+ # formula function call, leave it untouched. Formula-function
927
+ # terms (``log(x)``, ``sqrt(z)``, ``poly(x, 2)``, ``I(x ** 2)``)
928
+ # contain parentheses and are processed below even when their
929
+ # inner expressions contain arithmetic operators like ``**``.
930
+ if ("*" in term or ":" in term) and "(" not in term:
931
+ parts.append(term)
932
+ return
933
+
934
+ m = _FORMULA_FUNC_RE.match(term)
935
+ if not m:
936
+ parts.append(term)
937
+ return
938
+ func_name, inner = m.group(1), m.group(2).strip()
939
+
940
+ if func_name == "I":
941
+ try:
942
+ result = _eval_I_inner(inner, new_data)
943
+ except ValueError as exc:
944
+ raise ValueError(
945
+ f"Unsupported expression inside I(): {inner!r} ({exc})."
946
+ ) from exc
947
+ inner_var = _extract_single_var(inner)
948
+ if inner_var is None:
949
+ raise ValueError(
950
+ f"I() inner expression must reference a single column; "
951
+ f"got {inner!r}."
952
+ )
953
+ new_data[inner_var] = result
954
+ transformed.append((term, "I", inner_var))
955
+ parts.append(inner_var)
956
+ return
957
+
958
+ result = _apply_formula_function(func_name, inner, None, new_data)
959
+ inner_var = _extract_single_var(inner.split(",")[0])
960
+ if inner_var is None:
961
+ raise ValueError(
962
+ f"Formula function {func_name}() must have a single column "
963
+ f"as its first argument; got {inner!r}."
964
+ )
965
+ new_data[inner_var] = result
966
+ transformed.append((term, func_name, inner_var))
967
+ parts.append(inner_var)
968
+
969
+ for term in raw_terms:
970
+ _process_term(term, new_x_parts)
971
+
972
+ new_main = f"{y_part} ~ {' + '.join(new_x_parts)}"
973
+
974
+ if given_part is not None:
975
+ given_terms = _split_formula_terms(given_part, "+")
976
+ new_given_parts: List[str] = []
977
+ for term in given_terms:
978
+ _process_term(term, new_given_parts)
979
+ new_formula = f"{new_main} | {' + '.join(new_given_parts)}"
980
+ else:
981
+ new_formula = new_main
982
+
983
+ return new_data, new_formula, transformed
984
+
985
+
986
+ # Whitelisted operators for I() inner expressions. Only single-column
987
+ # references plus arithmetic on them are allowed.
988
+ _I_ALLOWED_BINOPS = {
989
+ ast.Add: lambda a, b: a + b,
990
+ ast.Sub: lambda a, b: a - b,
991
+ ast.Mult: lambda a, b: a * b,
992
+ ast.Div: lambda a, b: a / b,
993
+ ast.Pow: lambda a, b: a ** b,
994
+ ast.Mod: lambda a, b: a % b,
995
+ }
996
+
997
+
998
+ def _eval_I_inner(inner: str, data: pd.DataFrame):
999
+ """Safely evaluate an ``I()`` inner expression.
1000
+
1001
+ Allowed forms:
1002
+ - ``x``: just a column.
1003
+ - ``x ** k``, ``x * k``, ``x + k``, ``x - k``, ``x / k`` with a
1004
+ numeric constant ``k``.
1005
+ - ``x ** k + c``, ``x * k + c``, etc. (column * power + constant).
1006
+ - ``x + y`` (column + column) -- accepted because R allows it.
1007
+ Returns a numpy array.
1008
+
1009
+ We deliberately reject anything that looks like a function call inside
1010
+ ``I()`` (use the explicit ``log(x)`` syntax for that).
1011
+ """
1012
+ tree = ast.parse(inner, mode="eval")
1013
+ return _eval_I_node(tree.body, data)
1014
+
1015
+
1016
+ def _eval_I_node(node, data: pd.DataFrame):
1017
+ if isinstance(node, ast.Name):
1018
+ if node.id not in data.columns:
1019
+ raise ValueError(f"Unknown column {node.id!r}")
1020
+ return data[node.id].to_numpy()
1021
+ if isinstance(node, ast.Constant):
1022
+ if not isinstance(node.value, (int, float)):
1023
+ raise ValueError(
1024
+ f"Only numeric constants are allowed in I(); got {node.value!r}"
1025
+ )
1026
+ return node.value
1027
+ if isinstance(node, ast.BinOp):
1028
+ op_type = type(node.op)
1029
+ if op_type not in _I_ALLOWED_BINOPS:
1030
+ raise ValueError(
1031
+ f"Operator {op_type.__name__} is not allowed inside I()."
1032
+ )
1033
+ left = _eval_I_node(node.left, data)
1034
+ right = _eval_I_node(node.right, data)
1035
+ return _I_ALLOWED_BINOPS[op_type](left, right)
1036
+ if isinstance(node, ast.UnaryOp):
1037
+ if isinstance(node.op, ast.USub):
1038
+ return -_eval_I_node(node.operand, data)
1039
+ if isinstance(node.op, ast.UAdd):
1040
+ return _eval_I_node(node.operand, data)
1041
+ raise ValueError(
1042
+ f"Unary operator {type(node.op).__name__} not allowed in I()."
1043
+ )
1044
+ raise ValueError(
1045
+ f"Expressions of type {type(node).__name__} are not allowed in I()."
1046
+ )
1047
+
1048
+
1049
+ def _extract_single_var(inner: str) -> Optional[str]:
1050
+ """Return the single column name referenced in ``inner``.
1051
+
1052
+ Accepts forms: ``x``, ``x ** 2``, ``x + 1`` (column reference must be
1053
+ the LHS identifier; constants and arithmetic are allowed alongside it).
1054
+ Returns None if the expression does not reference exactly one column.
1055
+ """
1056
+ try:
1057
+ tree = ast.parse(inner, mode="eval")
1058
+ except SyntaxError:
1059
+ return None
1060
+ vars_found = set()
1061
+ for sub in ast.walk(tree):
1062
+ if isinstance(sub, ast.Name):
1063
+ vars_found.add(sub.id)
1064
+ # Filter out Python built-in names that may appear in the constant part.
1065
+ vars_found.discard("pi")
1066
+ vars_found.discard("e")
1067
+ if len(vars_found) == 1:
1068
+ return next(iter(vars_found))
1069
+ return None
1070
+
1071
+
1072
+ # ---------------------------------------------------------------------------
1073
+ # Multivariate slotting / auto-binning for slot 2+ numeric predictors
1074
+ # (R-flexplot ``flexplot_break_me`` parity, v0.8.x+).
1075
+ # After the formula is split, any numeric predictor in slot 2+ or in
1076
+ # ``given`` with more than ``bins`` unique values is binned into a
1077
+ # ``<varname>_binned`` column (used as the color / group aesthetic).
1078
+ # ---------------------------------------------------------------------------
1079
+
1080
+
1081
+ _DEFAULT_BINS = 3
1082
+
1083
+
1084
+ def _slot_bin_numeric_predictors(
1085
+ data: pd.DataFrame,
1086
+ formula: str,
1087
+ bins: Optional[int] = None,
1088
+ breaks: Optional[List[float]] = None,
1089
+ labels: Optional[List[str]] = None,
1090
+ ):
1091
+ """Bin numeric slot-2+ / given predictors into ``<name>_binned`` columns.
1092
+
1093
+ Mirrors R's ``flexplot_break_me``: a numeric predictor is binned when
1094
+ it appears in slot 2 or later (i.e. it's the color/group or a panel
1095
+ variable) AND its unique-value count exceeds ``bins``. Returns
1096
+ ``(new_data, new_color, new_given, bin_count)`` where ``new_data``
1097
+ has the new ``<varname>_binned`` columns appended (or overwritten),
1098
+ ``new_color`` is the column name to use for the color aesthetic
1099
+ (possibly the binned version), and ``new_given`` is the list of
1100
+ column names to use for facet variables (also possibly the binned
1101
+ versions). ``bin_count`` is the number of variables that were
1102
+ actually binned.
1103
+
1104
+ If a third non-given predictor is present (e.g. ``y ~ x1 + x2 + x3``),
1105
+ this raises ``ValueError`` because R limits the display to at most
1106
+ 4 visual variables (1 outcome + 3 predictors) and adding more
1107
+ predictors overwhelms the plot.
1108
+
1109
+ Parameters
1110
+ ----------
1111
+ bins : int, default 3
1112
+ Cut-point count used by ``pd.cut`` for the auto-bin path. Mirrors
1113
+ R's default ``bins=3``. Ignored when ``breaks`` is provided.
1114
+ breaks, labels : optional
1115
+ Reserved for forward-compat with R's explicit ``breaks=list(...)``
1116
+ API; current implementation accepts only a single breaks/labels
1117
+ pair (used for any predictor that needs binning). Tests pass a
1118
+ flat ``breaks`` and ``labels`` and rely on the auto-cut fallback
1119
+ path; richer dict-based breaks can be added in a later release.
1120
+ """
1121
+ bins = bins if bins is not None else _DEFAULT_BINS
1122
+ if not isinstance(bins, int) or bins < 2:
1123
+ # Silently fall back to default for invalid bins (callers should
1124
+ # have validated via _validate_binning_params already).
1125
+ bins = _DEFAULT_BINS
1126
+
1127
+ variables = parse_flexplot_formula(formula)
1128
+ if variables.get("intercept_only", False):
1129
+ return data.copy(), variables.get("color"), variables.get("given", []), 0
1130
+
1131
+ color = variables.get("color")
1132
+ given = variables.get("given", [])
1133
+ all_x = variables.get("all_x", []) # includes any +color tokens
1134
+
1135
+ # If there are 3+ non-given predictors (i.e. x + color + extra), reject.
1136
+ # R limits flexplot to 4 display vars total (1 outcome + 3 predictors)
1137
+ # for cognitive load; an extra slot would also need a 4th aesthetic.
1138
+ # `all_x` already includes interaction-expanded terms, so we strip
1139
+ # those out for the count.
1140
+ atom_predictors = [
1141
+ t for t in all_x if ":" not in t and "*" not in t
1142
+ ]
1143
+ if len(atom_predictors) > 2:
1144
+ raise ValueError(
1145
+ f"Formula {formula!r} has {len(atom_predictors)} non-given "
1146
+ f"predictors ({atom_predictors}); flexplot supports at most "
1147
+ f"two non-given predictors (x and an optional color). For "
1148
+ f"more, move them into the `| given` part of the formula."
1149
+ )
1150
+
1151
+ new_data = data.copy()
1152
+ bin_count = 0
1153
+
1154
+ # Helper: bin a numeric column into ``<col>_binned`` using ``pd.cut``
1155
+ # over equal-width cuts between min/max.
1156
+ def _bin_one(col: str) -> Optional[str]:
1157
+ nonlocal bin_count
1158
+ if col is None or col not in new_data.columns:
1159
+ return col
1160
+ s = new_data[col]
1161
+ if not pd.api.types.is_numeric_dtype(s):
1162
+ return col
1163
+ n_unique = s.dropna().nunique()
1164
+ if n_unique <= bins:
1165
+ # Already low-cardinality: skip binning (R skips when <= bins).
1166
+ return col
1167
+ # If we got an explicit breaks list, use it; otherwise equal-width.
1168
+ x_min = float(np.nanmin(s.to_numpy()))
1169
+ x_max = float(np.nanmax(s.to_numpy()))
1170
+ cuts = np.linspace(x_min, x_max, num=int(bins) + 1).tolist()
1171
+ binned_col = f"{col}_binned"
1172
+ new_data[binned_col] = pd.cut(
1173
+ s, bins=cuts, labels=labels, include_lowest=True
1174
+ ).astype(str)
1175
+ bin_count += 1
1176
+ return binned_col
1177
+
1178
+ # Bin the color predictor if it's numeric and high-cardinality.
1179
+ new_color = _bin_one(color) if color is not None else None
1180
+
1181
+ # Bin the given variables similarly.
1182
+ new_given = [_bin_one(g) if g is not None else None for g in given]
1183
+ new_given = [g for g in new_given if g is not None]
1184
+
1185
+ return new_data, new_color, new_given, bin_count
1186
+
1187
+
1188
+ # ---------------------------------------------------------------------------
1189
+ # flexplot_alpha_default / match_jitter_categorical parity (v0.8.x+).
1190
+ # ---------------------------------------------------------------------------
1191
+
1192
+
1193
+ # Sentinel alpha used internally to mean "user did not pass alpha"
1194
+ # (matches R's ``alpha=.99977`` default in flexplot_prep_variables).
1195
+ _FLEXPLOT_ALPHA_SENTINEL = 0.99977
1196
+
1197
+
1198
+ def flexplot_alpha_default(data: pd.DataFrame, x: Optional[str], y: str,
1199
+ alpha: Optional[float]) -> float:
1200
+ """Return the alpha to use for the raw-data geom.
1201
+
1202
+ Mirrors R's ``flexplot_alpha_default``:
1203
+ - If user explicitly set alpha (not the sentinel), return it as-is.
1204
+ - Otherwise (R sentinel ``.99977`` / Python ``None``), use 0.2 for
1205
+ categorical x, 0.5 for numeric x.
1206
+ - For intercept-only formulas (no x axis), pass through.
1207
+
1208
+ The Python-side alpha resolution (see ``flexplot()``) keeps the legacy
1209
+ behavior of ``alpha=0.3`` for numeric binary y; this helper is the
1210
+ categorical-vs-numeric split.
1211
+ """
1212
+ if alpha is not None and alpha != _FLEXPLOT_ALPHA_SENTINEL:
1213
+ # Explicit user value.
1214
+ return float(alpha)
1215
+ if x is None or x not in data.columns:
1216
+ # Intercept-only / no x: pass through.
1217
+ return float(alpha) if alpha is not None else 0.5
1218
+ s = data[x]
1219
+ if not pd.api.types.is_numeric_dtype(s):
1220
+ return 0.2
1221
+ return 0.5
1222
+
1223
+
1224
+ def match_jitter_categorical(x, is_categorical: bool):
1225
+ """Resolve the jitter argument for the categorical / mixed-x branch.
1226
+
1227
+ Mirrors R's ``match_jitter_categorical``:
1228
+
1229
+ - ``None`` + categorical x → (0.2, 0)
1230
+ - ``None`` + numeric x → (0, 0) (no jitter)
1231
+ - ``True`` → (0.2, 0)
1232
+ - ``False`` → (0, 0)
1233
+ - numeric length 1 → (x, 0)
1234
+ - numeric length 2 → (x, y)
1235
+ - anything else → raises ``ValueError``
1236
+
1237
+ Returns a 2-tuple ``(width, height)``.
1238
+ """
1239
+ if x is None:
1240
+ return (0.2, 0.0) if is_categorical else (0.0, 0.0)
1241
+ if isinstance(x, bool):
1242
+ return (0.2, 0.0) if x else (0.0, 0.0)
1243
+ if isinstance(x, (int, float)):
1244
+ return (float(x), 0.0)
1245
+ if isinstance(x, (list, tuple)):
1246
+ if len(x) == 1:
1247
+ return (float(x[0]), 0.0)
1248
+ if len(x) == 2:
1249
+ return (float(x[0]), float(x[1]))
1250
+ if len(x) > 2:
1251
+ raise ValueError(
1252
+ f"jitter must be a length-1 or length-2 sequence; got length "
1253
+ f"{len(x)}: {x!r}."
1254
+ )
1255
+ # Empty sequence
1256
+ raise ValueError(f"jitter must be non-empty; got {x!r}.")
1257
+ raise ValueError(
1258
+ f"jitter must be None, a bool, or a numeric length-1/2 sequence; "
1259
+ f"got {type(x).__name__}: {x!r}."
1260
+ )
1261
+
1262
+
1263
+ def flexplot(
1264
+ formula: str,
1265
+ data: pd.DataFrame,
1266
+ method: str = "auto",
1267
+ random_effects: Optional[str] = None,
1268
+ mixed_backend: str = "auto",
1269
+ uncertainty: Optional[str] = "ci",
1270
+ level: float = 0.95,
1271
+ bands: Optional[List[float]] = None,
1272
+ overlay: Optional[List] = None,
1273
+ bins: Optional[int] = None,
1274
+ labels: Optional[List[str]] = None,
1275
+ breaks: Optional[List[float]] = None,
1276
+ spread: Optional[str] = None,
1277
+ sample: Optional[int] = None,
1278
+ ghost_line: Optional[str] = None,
1279
+ plot_type: Optional[str] = None,
1280
+ return_data: bool = False,
1281
+ ghost_reference=None,
1282
+ plot_string=None,
1283
+ related: bool = False,
1284
+ interaction_model: bool = False,
1285
+ jitter: Optional[Union[bool, List[float]]] = None,
1286
+ alpha: Optional[float] = None,
1287
+ raw_data: bool = True,
1288
+ **kwargs,
1289
+ ):
1290
+ """Intelligent multivariate graphics via formulas.
1291
+
1292
+ Parameters
1293
+ ----------
1294
+ formula : str
1295
+ Flexplot formula of the form ``y ~ x [+ color] [| given1, given2]``.
1296
+ R-style interaction syntax (``y ~ x*z``, ``y ~ x:z``) is also
1297
+ accepted since v0.6.2; see Notes below.
1298
+ data : pd.DataFrame
1299
+ Non-empty data frame holding the referenced columns.
1300
+ method : {"auto", "lm", "loess", "quadratic", "polynomial", "cubic", "logistic", "rlm", "poisson", "Gamma", "mixedlm", "lmer", "glmer"}
1301
+ Smoother for the numeric-vs-numeric branch. ``"auto"`` selects LM.
1302
+ ``"quadratic"`` / ``"polynomial"``: degree-2 OLS in x (matches R).
1303
+ ``"cubic"``: degree-3 OLS in x.
1304
+ ``"logistic"``: GLM with logit link on numeric binary y.
1305
+ ``"rlm"``: robust regression via statsmodels RLM (Huber).
1306
+ ``"poisson"``: GLM with log link (requires non-negative y).
1307
+ ``"Gamma"``: GLM with inverse link (requires strictly positive y).
1308
+ ``"mixedlm"`` / ``"lmer"``: linear mixed-effects model with a
1309
+ random intercept (and optional random slope via
1310
+ ``random_effects='(1 + x|group)'``).
1311
+ ``"glmer"``: binomial mixed-effects model (random intercept).
1312
+ Non-conforming outcomes for logistic/poisson/Gamma fall back to OLS
1313
+ with a ``UserWarning``.
1314
+ random_effects : str, optional
1315
+ Random-effects spec for mixed methods:
1316
+ - Column name (e.g., ``"school"``) => random intercept ``(1|school)``.
1317
+ - lme4-style mini spec (e.g., ``"(1|school)"``,
1318
+ ``"(1 + x|school)"``).
1319
+ Required when ``method`` is ``"mixedlm"``, ``"lmer"``, or
1320
+ ``"glmer"``.
1321
+ mixed_backend : {"auto", "statsmodels"}, default "auto"
1322
+ Mixed-model backend selector. ``"auto"`` currently resolves to
1323
+ statsmodels; this argument is reserved for future ``pymer4``/R
1324
+ integration.
1325
+ uncertainty : {None, "ci", "prediction", "bootstrap"}, default "ci"
1326
+ Type of uncertainty band drawn around the fitted line.
1327
+ - ``None``: no fit, just the scatter.
1328
+ - ``"ci"``: confidence interval on the mean response (plotnine default).
1329
+ - ``"prediction"``: residual-based prediction interval on new observations.
1330
+ - ``"bootstrap"``: case-resampled CI (loess branch only; n_resamples=200).
1331
+ level : float in (0, 1), default 0.95
1332
+ Coverage probability for a single band. Ignored when ``bands`` is given.
1333
+ bands : list of float in (0, 1), optional
1334
+ Nested coverage levels (e.g., ``[0.5, 0.8, 0.95]``) for Tufte-style
1335
+ multi-ribbon display. Overrides ``level`` when provided.
1336
+ overlay : list of str or dict, optional
1337
+ Additional smoother specs to overlay on the same axes alongside the
1338
+ primary ``method``. Each entry is either a method name (``"lm"``,
1339
+ ``"loess"``, ``"rlm"``, etc.) or a dict with keys:
1340
+ - ``method`` (required): one of the recognized smoother methods.
1341
+ - ``color`` (optional, default cycles through a 5-color palette).
1342
+ - ``label`` (optional, default = ``method``): legend label.
1343
+ - ``uncertainty`` (optional, default ``"ci"``): per-overlay band type.
1344
+ - ``level`` (optional, default 0.95): per-overlay band coverage.
1345
+ bins : int, optional
1346
+ Discretize a numeric x into ``bins`` equal-width intervals before
1347
+ plotting, so the discrete-style summary (geom_jitter + dispersion
1348
+ marker) applies. Mutually exclusive with ``breaks`` (which wins).
1349
+ No-op when x is already discrete or non-numeric.
1350
+ labels : list of str, optional
1351
+ Custom labels for the discrete x levels produced by ``bins`` /
1352
+ ``breaks``. Length must equal ``bins`` (when given with bins) or
1353
+ ``len(breaks) - 1`` (when given with breaks).
1354
+ breaks : list of float, optional
1355
+ Explicit cut points for discretizing numeric x. Takes precedence
1356
+ over ``bins`` when both are given (a ``UserWarning`` is emitted).
1357
+ spread : {None, "ci", "sterr", "stdev", "range", "iqr", "quartiles", "no"}, default None
1358
+ Dispersion marker drawn in the discrete-x branch alongside
1359
+ ``geom_jitter``. Mirrors R-flexplot's ``spread``.
1360
+ - ``None`` / ``"quartiles"`` / ``"iqr"``: median ± Q1/Q3 IQR
1361
+ (R's default for discrete x).
1362
+ - ``"ci"``: bootstrap CI on the mean.
1363
+ - ``"sterr"``: mean ± 1.96 × standard error of the mean.
1364
+ - ``"stdev"``: mean ± 1 SD as a pointrange.
1365
+ - ``"range"``: min-max range.
1366
+ - ``"no"``: no summary layer at all.
1367
+ sample : int, optional
1368
+ Subsample N rows for the plotnine layers (scatter / jitter) while
1369
+ keeping the smoother fits on the full DataFrame. No-op when
1370
+ ``N >= len(data)``. Deterministic via ``np.random.default_rng(0)``.
1371
+ ghost_line : {"red", "dashed", "slope1", None}, default None
1372
+ Reference line drawn after the main layers. ``"red"`` for a solid
1373
+ red threshold at y=0; ``"dashed"`` for a black dashed reference
1374
+ at y=0; ``"slope1"`` for a diagonal slope=1 reference line for
1375
+ prediction-vs-observed overlays (v0.7.3+).
1376
+ plot_type : {"scatter", "line", "boxplot", "bar", "histogram", "qq", "density", "violin", None}, default None
1377
+ Explicit geom override. Bypasses the auto-dispatch. For
1378
+ intercept-only formulas, ``"histogram"`` (default), ``"qq"``,
1379
+ ``"density"``, ``"boxplot"``, and ``"violin"`` produce univariate
1380
+ distribution plots.
1381
+ return_data : bool, default False
1382
+ When ``True``, return ``{"plot": ggplot, "data": DataFrame}``
1383
+ instead of just the plot. Useful with ``sample=`` to know which
1384
+ rows were plotted.
1385
+ ghost_reference : pd.DataFrame, optional
1386
+ Reference dataset to overlay on the same axes. Two patterns:
1387
+ - Columns ``(x, y)``: draws a gray geom_point layer (reference scatter).
1388
+ - Columns ``(x, "pred")``: draws a red dashed geom_line (prediction line).
1389
+ plot_string : dict, optional
1390
+ Override the axis/legend labels derived from the formula. Accepts
1391
+ keys ``x``, ``y``, ``title``, ``subtitle``, ``caption``, ``color``.
1392
+ related : bool, default False
1393
+ R-flexplot's paired-samples flag. When True and the formula is
1394
+ ``y ~ x`` with a two-level categorical predictor, the plot shows
1395
+ paired difference scores (level2 - level1) against x = 1, with a
1396
+ reference line at 0 and a dispersion marker. Requires equal group
1397
+ sizes and no color or panel variables. Raises ``ValueError`` when
1398
+ the precondition is not met.
1399
+ interaction_model : bool, default False
1400
+ When ``True`` and the formula contains ``*`` or ``:`` syntax, fit
1401
+ a statsmodels OLS with the actual interaction term and overlay
1402
+ non-parallel per-color-group regression lines (rather than the
1403
+ default additive fit with parallel slopes). Suppresses the
1404
+ "additive fit" UserWarning when set. Falls back to the additive
1405
+ path when the formula has no interaction term, no separate color
1406
+ group, or only one color level.
1407
+ **kwargs
1408
+ Reserved for future extension.
1409
+
1410
+ Returns
1411
+ -------
1412
+ plotnine.ggplot
1413
+ The composed plot object. Call ``.draw()`` to render or ``.save()``
1414
+ to write to disk.
1415
+
1416
+ Notes
1417
+ -----
1418
+ For the numeric-vs-binary branch (binomial GLM), the band is always drawn
1419
+ on the response (probability) scale; plotnine handles the inverse-link
1420
+ transformation internally. Numeric binary ``[0, 1]`` y (v0.6.1+) and
1421
+ string binary y both route to the binomial branch. Explicit
1422
+ ``method="logistic"`` bypasses the binary pre-check and forces the
1423
+ numeric branch with a parametric logistic GLM.
1424
+
1425
+ Interaction syntax (``*``, ``:``) is parsed since v0.6.2. The default
1426
+ fit remains **additive** (parallel slopes per color group); a
1427
+ ``UserWarning`` is emitted whenever interaction syntax is detected.
1428
+ Pass ``interaction_model=True`` (v0.7.0+) to fit the actual
1429
+ interaction term and overlay non-parallel per-color-group regression
1430
+ lines; this also suppresses the additive-fit warning.
1431
+
1432
+ Examples
1433
+ --------
1434
+ >>> import pandas as pd
1435
+ >>> import numpy as np
1436
+ >>> from pyflexplot import flexplot
1437
+ >>> rng = np.random.default_rng(0)
1438
+ >>> df = pd.DataFrame({"x": rng.normal(size=100), "y": rng.normal(size=100)})
1439
+ >>> p = flexplot("y ~ x", data=df)
1440
+ >>> isinstance(p, ggplot)
1441
+ True
1442
+
1443
+ With uncertainty bands:
1444
+
1445
+ >>> p = flexplot("y ~ x", data=df, bands=[0.5, 0.8, 0.95])
1446
+ >>> any(isinstance(layer.geom, geom_smooth) for layer in p.layers)
1447
+ True
1448
+
1449
+ With overlay smoothers:
1450
+
1451
+ >>> p = flexplot(
1452
+ ... "y ~ x", data=df,
1453
+ ... overlay=[{"method": "loess", "label": "LOESS smoother"}],
1454
+ ... )
1455
+ >>> smooth_layers = [l for l in p.layers if isinstance(l.geom, geom_smooth)]
1456
+ >>> len(smooth_layers) >= 2
1457
+ True
1458
+
1459
+ Auto-binning numeric x (v0.6.4):
1460
+
1461
+ >>> df2 = pd.DataFrame({
1462
+ ... "x": rng.uniform(0, 100, size=80),
1463
+ ... "y": rng.normal(size=80),
1464
+ ... })
1465
+ >>> p = flexplot("y ~ x", data=df2, bins=4)
1466
+ >>> any(isinstance(layer.geom, geom_jitter) for layer in p.layers)
1467
+ True
1468
+
1469
+ Polynomial fit on a non-linear signal (v0.6.4):
1470
+
1471
+ >>> df3 = pd.DataFrame({
1472
+ ... "x": np.linspace(-3, 3, 60),
1473
+ ... "y": np.linspace(-3, 3, 60) ** 2 + rng.normal(scale=0.3, size=60),
1474
+ ... })
1475
+ >>> p = flexplot("y ~ x", data=df3, method="polynomial")
1476
+ >>> any(isinstance(layer.geom, geom_line) for layer in p.layers)
1477
+ True
1478
+ """
1479
+ if method not in _VALID_FLEXPLOT_METHODS:
1480
+ raise ValueError(
1481
+ f"method must be one of {sorted(_VALID_FLEXPLOT_METHODS)}; got {method!r}. "
1482
+ "Pass 'auto' for the default behaviour (LM for numeric-vs-numeric, "
1483
+ "binomial GLM for numeric-vs-binary)."
1484
+ )
1485
+ validate_uncertainty_params(uncertainty, level, bands, method)
1486
+ overlay_specs = _normalize_overlay(overlay)
1487
+ if spread is not None and spread not in (s for s in _VALID_SPREAD if s):
1488
+ raise ValueError(
1489
+ f"spread must be one of {sorted(s for s in _VALID_SPREAD if s)} "
1490
+ f"or None; got {spread!r}."
1491
+ )
1492
+
1493
+ variables = parse_flexplot_formula(formula)
1494
+ if variables.get("has_interaction") and not interaction_model:
1495
+ # v0.6.x default: parser accepts interaction syntax (R-compatible)
1496
+ # but the fit remains additive (parallel slopes per color group).
1497
+ # Warn so users aren't misled. When interaction_model=True is
1498
+ # explicit, the fit uses the actual interaction term via
1499
+ # _add_interaction_smooth() and no warning is needed (v0.7.0+).
1500
+ warnings.warn(
1501
+ f"Interaction syntax detected in formula {formula!r} but flexplot's "
1502
+ f"default fit is additive (parallel slopes per color group). "
1503
+ f"Pass `interaction_model=True` for true non-parallel slopes. "
1504
+ f"To suppress this warning, write the formula without `*` or `:`.",
1505
+ UserWarning,
1506
+ stacklevel=2,
1507
+ )
1508
+
1509
+ # --- Formula-function evaluation (R ``formula_functions`` parity) ---
1510
+ # Detect terms like ``log(x)``, ``sqrt(x)``, ``poly(x, 2)``, ``I(x**2)``
1511
+ # in the right-hand side, apply the whitelisted function to ``data``,
1512
+ # store the result in a column named after the inner variable, and
1513
+ # rewrite the formula so downstream code sees the simpler name.
1514
+ transformed_data, transformed_formula, transformed_terms = (
1515
+ _apply_formula_functions(formula, data)
1516
+ )
1517
+ if transformed_terms:
1518
+ # Re-parse the rewritten formula; variables dict now reflects the
1519
+ # simpler (un-transed) term names.
1520
+ variables = parse_flexplot_formula(transformed_formula)
1521
+ # Preserve the interaction flag from the original parse (it was
1522
+ # already detected before the function-rewrite pass).
1523
+ if "has_interaction" not in variables:
1524
+ variables["has_interaction"] = False
1525
+ formula = transformed_formula
1526
+ data = transformed_data
1527
+
1528
+ _validate_data_for_plot(
1529
+ formula, data, variables, intercept_only=variables.get("intercept_only", False)
1530
+ )
1531
+
1532
+ y = variables["y"]
1533
+ x = variables["x"]
1534
+ color = variables["color"]
1535
+ given = variables["given"]
1536
+ intercept_only = variables.get("intercept_only", False)
1537
+
1538
+ # --- Auto-categorize low-cardinality numeric predictors ---
1539
+ # R-flexplot's ``convert_if_less_than_five`` turns numeric variables
1540
+ # with <5 unique values into ordered factors. This must run BEFORE
1541
+ # the type-detection / binning below so the low-cardinality numeric
1542
+ # path is taken (discrete x branch) rather than the high-cardinality
1543
+ # numeric path (LM / loess smoother).
1544
+ if not intercept_only and x is not None:
1545
+ cat_targets = [v for v in [x, color, *given] if v is not None]
1546
+ # Avoid converting columns that already are transformed to a
1547
+ # ``_binned`` string — those are intentionally stringified above.
1548
+ cat_targets = [v for v in cat_targets if not v.endswith("_binned")]
1549
+ if cat_targets:
1550
+ data = _convert_low_cardinality_to_categorical(data, cat_targets)
1551
+
1552
+ # --- Optional subsampling (v0.6.5+) ---
1553
+ # When ``sample=N`` is set and N < len(data), subsample to N rows for
1554
+ # plotting only. Subsequent smoother fits still use the FULL data so the
1555
+ # fit isn't degraded by the subsample. We track ``_sampled_df`` for
1556
+ # return_data= so the caller knows which rows were plotted.
1557
+ if sample is not None:
1558
+ if not isinstance(sample, int) or isinstance(sample, bool):
1559
+ raise TypeError(
1560
+ f"sample must be an int >= 1; got {type(sample).__name__} ({sample!r})."
1561
+ )
1562
+ if sample < 1:
1563
+ raise ValueError(f"sample must be >= 1; got {sample}.")
1564
+ if sample is not None and sample < len(data):
1565
+ rng_sample = np.random.default_rng(0) # deterministic for reproducibility
1566
+ sampled_idx = rng_sample.choice(len(data), size=sample, replace=False)
1567
+ sampled_idx = np.sort(sampled_idx)
1568
+ plot_input_df = data.iloc[sampled_idx].reset_index(drop=True)
1569
+ fit_input_df = data # full data; smoother fits unchanged
1570
+ else:
1571
+ plot_input_df = data
1572
+ fit_input_df = data
1573
+
1574
+ # --- Multivariate slotting / auto-binning for slot 2+ numeric predictors ---
1575
+ # Mirrors R's ``flexplot_break_me``: a numeric color or given variable
1576
+ # with more than ``bins`` unique values is binned into ``<name>_binned``
1577
+ # and that column drives the color aesthetic / facets. Re-raises on
1578
+ # formulas with 3+ non-given predictors.
1579
+ if not intercept_only:
1580
+ slot_data, binned_color, binned_given, _bin_count = (
1581
+ _slot_bin_numeric_predictors(
1582
+ fit_input_df, formula, bins=bins, breaks=breaks, labels=labels
1583
+ )
1584
+ )
1585
+ # Apply the binning to both plot and fit inputs.
1586
+ binned_cols = [
1587
+ c for c in slot_data.columns
1588
+ if c not in fit_input_df.columns and c.endswith("_binned")
1589
+ ]
1590
+ # The helper returns ``slot_data`` containing the original columns
1591
+ # PLUS any ``<name>_binned`` ones. We merge them back into both
1592
+ # ``plot_input_df`` and ``fit_input_df`` so the aesthetic mappings
1593
+ # and the smoother fits see the binned columns.
1594
+ if binned_cols:
1595
+ for col in binned_cols:
1596
+ # Original source column for the binned copy.
1597
+ src = col[: -len("_binned")]
1598
+ if src in fit_input_df.columns:
1599
+ # Carry over the binned values from slot_data; they are
1600
+ # a deterministic function of the original numeric
1601
+ # column, so the alignment is index-based.
1602
+ plot_input_df[col] = slot_data[col].to_numpy()
1603
+ fit_input_df[col] = slot_data[col].to_numpy()
1604
+ # Update variables' color / given to point at the binned cols
1605
+ # so the aes / facet wiring below uses them.
1606
+ if binned_color is not None and color is not None:
1607
+ variables["color"] = binned_color
1608
+ color = binned_color
1609
+ if binned_given:
1610
+ # Replace ``given`` with the binned variants while keeping
1611
+ # the same length / ordering as the original list.
1612
+ new_given: List[Optional[str]] = []
1613
+ for orig in variables["given"]:
1614
+ if orig is None:
1615
+ new_given.append(None)
1616
+ continue
1617
+ candidate = f"{orig}_binned"
1618
+ if candidate in fit_input_df.columns:
1619
+ new_given.append(candidate)
1620
+ else:
1621
+ new_given.append(orig)
1622
+ variables["given"] = new_given
1623
+ given = new_given
1624
+
1625
+ if intercept_only:
1626
+ # Intercept-only: show a univariate distribution of y.
1627
+ # R-flexplot supports histogram/qq/density/boxplot/violin via plot.type.
1628
+ p = _plot_univariate(plot_input_df, y, plot_type=plot_type, bins=bins)
1629
+ if return_data:
1630
+ return {"plot": p, "data": plot_input_df}
1631
+ return p
1632
+
1633
+ if not isinstance(related, bool):
1634
+ raise TypeError(f"related must be a bool; got {type(related).__name__}.")
1635
+
1636
+ # --- Related-samples / paired difference plot (R-flexplot related=T) ---
1637
+ # Only valid for y ~ x where x is a two-level grouping variable and there
1638
+ # are no color/given facets. We replace the data with paired difference
1639
+ # scores (level2 - level1) and draw a univariate difference plot.
1640
+ if related:
1641
+ if color is not None or len(given) > 0:
1642
+ raise ValueError(
1643
+ "related=True is only supported for formulas with a single "
1644
+ "predictor and no color or panel variables (e.g., 'y ~ x')."
1645
+ )
1646
+ if x is None:
1647
+ raise ValueError("related=True requires a predictor variable.")
1648
+
1649
+ x_series = plot_input_df[x]
1650
+ # Ensure a categorical-style grouping variable with exactly two levels.
1651
+ if pd.api.types.is_numeric_dtype(x_series) and x_series.nunique(dropna=True) == 2:
1652
+ x_series = x_series.astype(str)
1653
+ levs = sorted(x_series.dropna().unique())
1654
+ if len(levs) != 2:
1655
+ raise ValueError(
1656
+ f"related=True requires exactly 2 levels of the predictor; "
1657
+ f"{x!r} has {len(levs)} levels."
1658
+ )
1659
+
1660
+ groups = {
1661
+ lev: plot_input_df.loc[x_series == lev, y].reset_index(drop=True)
1662
+ for lev in levs
1663
+ }
1664
+ sizes = [len(g) for g in groups.values()]
1665
+ if len(set(sizes)) != 1:
1666
+ raise ValueError(
1667
+ "related=True requires equal group sizes to compute paired "
1668
+ f"differences; got sizes {dict(zip(levs, sizes))}."
1669
+ )
1670
+
1671
+ diff_label = f"Difference ({levs[1]}-{levs[0]})"
1672
+ related_df = pd.DataFrame({diff_label: groups[levs[1]].to_numpy() - groups[levs[0]].to_numpy()})
1673
+ alpha_rel = alpha if alpha is not None else 0.5
1674
+ p = _plot_related(
1675
+ related_df,
1676
+ diff_label,
1677
+ spread,
1678
+ plot_type,
1679
+ jitter,
1680
+ alpha_rel,
1681
+ raw_data,
1682
+ )
1683
+ if return_data:
1684
+ return {"plot": p, "data": related_df}
1685
+ return p
1686
+
1687
+ # Reject 3+ given variables: the formula parser accepts them but only
1688
+ # two are actually used (facet_grid takes at most two). Better to fail
1689
+ # loudly than silently drop the third.
1690
+ if len(given) > 2:
1691
+ raise ValueError(
1692
+ f"Formula {formula!r} has {len(given)} given variables after '|': "
1693
+ f"{given}. flexplot supports at most 2 given variables "
1694
+ "(a row facet and a column facet)."
1695
+ )
1696
+
1697
+ # Determine variable types
1698
+ is_y_numeric = pd.api.types.is_numeric_dtype(data[y])
1699
+
1700
+ # Numeric-x binning: if bins / breaks / labels are provided and x is
1701
+ # numeric (and not already auto-discrete), discretize x so the
1702
+ # discrete-style summary applies. Validation:
1703
+ # - bins: int >= 2 (>= 2 needed to be meaningful).
1704
+ # - breaks: list of floats, len >= 2, monotonically increasing, span the
1705
+ # x range.
1706
+ # - labels: list of strings, len = len(breaks) - 1 when provided with
1707
+ # breaks, or len = bins when provided with bins.
1708
+ # Mutual precedence: breaks > bins (breaks overrides bins when both set).
1709
+ # Validation lives in _validate_binning_params.
1710
+ if not _is_discrete(plot_input_df[x]) and pd.api.types.is_numeric_dtype(plot_input_df[x]):
1711
+ _validate_binning_params(bins, labels, breaks, plot_input_df[x])
1712
+ plot_df, x_discretized = _maybe_bin_numeric_x(
1713
+ plot_input_df, x, bins=bins, labels=labels, breaks=breaks
1714
+ )
1715
+ if x_discretized:
1716
+ is_x_discrete = True
1717
+ else:
1718
+ plot_df = plot_input_df.copy()
1719
+ is_x_discrete = False
1720
+ else:
1721
+ plot_df = plot_input_df.copy()
1722
+ is_x_discrete = _is_discrete(plot_input_df[x])
1723
+ # Convert numeric discrete X to string/categorical so plotnine treats x-axis as discrete levels
1724
+ if is_x_discrete and pd.api.types.is_numeric_dtype(plot_df[x]):
1725
+ plot_df[x] = plot_df[x].astype(str)
1726
+
1727
+ # Binary-0/1 pre-check: a numeric y whose unique values are a subset of
1728
+ # {0, 1} should be treated as a binary outcome for binomial smoothing,
1729
+ # regardless of pandas' is_numeric_dtype (which returns True for int).
1730
+ # Without this pre-check, numeric binary y would fall into the LM/loess
1731
+ # branch and the binomial GLM branch would only fire for non-numeric y
1732
+ # (where the .astype(float) below would raise first).
1733
+ # BUT: explicit method='logistic' bypasses this and routes through the
1734
+ # numeric branch with a logistic GLM (see _add_parametric_smooth).
1735
+ y_is_binary = False
1736
+ if is_y_numeric and method not in {"logistic", "glmer"}:
1737
+ try:
1738
+ unique_y = pd.Series(plot_input_df[y].dropna().astype(float)).unique()
1739
+ except (ValueError, TypeError):
1740
+ unique_y = None
1741
+ y_is_binary = (
1742
+ unique_y is not None
1743
+ and len(unique_y) == 2
1744
+ and set(unique_y).issubset({0.0, 1.0})
1745
+ )
1746
+
1747
+ # Build the base aesthetic with color/group when needed so all geoms pick it up.
1748
+ aes_kwargs = {"x": x, "y": y}
1749
+ if color:
1750
+ # Convert numeric discrete color variable (like pclass = 1, 2, 3) to string/categorical
1751
+ # so plotnine uses a discrete color palette rather than a continuous gradient.
1752
+ if pd.api.types.is_numeric_dtype(plot_df[color]) and _is_discrete(plot_df[color]):
1753
+ plot_df[color] = plot_df[color].astype(str)
1754
+ aes_kwargs["color"] = color
1755
+ aes_kwargs["group"] = color
1756
+ p = ggplot(plot_df, aes(**aes_kwargs))
1757
+
1758
+ # --- Optional plot_type override (v0.6.5+) ---
1759
+ # Bypasses the auto-dispatch and forces a specific geom. Useful when
1760
+ # the user knows they want a boxplot regardless of how x is shaped, or
1761
+ # when the auto-dispatch picks the wrong branch because the data
1762
+ # violates heuristics (e.g. 11 unique values in x rather than 10).
1763
+ if plot_type is not None:
1764
+ if plot_type not in _VALID_PLOT_TYPES:
1765
+ raise ValueError(
1766
+ f"plot_type must be one of {sorted(_VALID_PLOT_TYPES)}; got {plot_type!r}."
1767
+ )
1768
+ if plot_type == "scatter":
1769
+ p += geom_point(alpha=0.5)
1770
+ elif plot_type == "line":
1771
+ p += geom_line()
1772
+ elif plot_type == "boxplot":
1773
+ p += geom_boxplot()
1774
+ elif plot_type == "violin":
1775
+ p += geom_violin()
1776
+ elif plot_type == "bar":
1777
+ # plotnine's geom_bar doesn't accept fun=; use stat_summary
1778
+ # with fun_y=np.mean + geom="bar" to get a bar chart of
1779
+ # group means per x level.
1780
+ p += stat_summary(fun_y=np.mean, geom="bar")
1781
+ # Skip the auto-dispatch below.
1782
+ skip_dispatch = True
1783
+ else:
1784
+ skip_dispatch = False
1785
+
1786
+ # --- jitter / alpha / raw_data resolution (v0.8.0+, R-parity) ---
1787
+ # R ``match_jitter_categorical``: None + categorical x -> (0.2, 0);
1788
+ # None + numeric x -> (0, 0); True -> (0.2, 0); False -> (0, 0);
1789
+ # numeric length-1 -> (x, 0); length-2 -> (x, y). We delegate to the
1790
+ # helper for the categorical-numeric split. The check uses the
1791
+ # POST-binning ``is_x_discrete`` (which is True when ``bins=`` /
1792
+ # ``breaks=`` discretized x, or when low-cardinality conversion kicked
1793
+ # in) so a user who asks for `bins=4` always gets the categorical
1794
+ # jitter defaults regardless of the underlying numeric dtype.
1795
+ is_x_discrete_for_jitter = bool(is_x_discrete)
1796
+ if isinstance(jitter, (list, tuple)) and len(jitter) == 2:
1797
+ # Explicit numeric pair: bypass the R rule so users can still get
1798
+ # the exact jitter widths they want.
1799
+ jitter_xy = (float(jitter[0]), float(jitter[1]))
1800
+ elif isinstance(jitter, (int, float)) and not isinstance(jitter, bool):
1801
+ # Numeric length-1: pass through (R: ``c(.2)`` -> ``(0.2, 0)``).
1802
+ jitter_xy = (float(jitter), 0.0)
1803
+ else:
1804
+ jitter_xy = match_jitter_categorical(jitter, is_x_discrete_for_jitter)
1805
+ # alpha: explicit value (float in (0, 1]) wins everywhere; otherwise
1806
+ # use flexplot_alpha_default's categorical/numeric rule (0.2 for
1807
+ # categorical x, 0.5 for numeric x). Numeric binary y keeps the
1808
+ # legacy 0.3 default (parity with prior releases and a slightly
1809
+ # softer overlay on the tight 0/1 cluster).
1810
+ if alpha is not None:
1811
+ if not isinstance(alpha, (int, float)) or not (0 < alpha <= 1):
1812
+ raise ValueError(
1813
+ f"alpha must be a float in (0, 1]; got {alpha!r}."
1814
+ )
1815
+ alpha_point = float(alpha)
1816
+ else:
1817
+ if y_is_binary:
1818
+ alpha_point = 0.3
1819
+ else:
1820
+ alpha_point = flexplot_alpha_default(plot_input_df, x, y, alpha)
1821
+
1822
+ # Determine plot type.
1823
+ # Order matters:
1824
+ # 1. Binary 0/1 y must be detected before the generic numeric branch
1825
+ # (otherwise int/float [0, 1] y falls into LM/loess).
1826
+ # 2. Numeric X is "discrete" when _is_discrete() returns True (numeric
1827
+ # with <=10 unique values; post-05ac368 R-flexplot parity).
1828
+ if not skip_dispatch and y_is_binary and not is_x_discrete:
1829
+ # Binomial GLM branch — numeric binary outcome with numeric x.
1830
+ if raw_data:
1831
+ p += geom_point(alpha=alpha_point)
1832
+ p = _add_binomial_smooth(p, data, x, y, uncertainty, level, bands, color=color)
1833
+ if overlay_specs:
1834
+ p = _add_overlay_binomial(p, data, x, y, overlay_specs)
1835
+
1836
+ elif not skip_dispatch and not is_y_numeric and not is_x_discrete:
1837
+ # Non-numeric y (string/categorical) with numeric x. Validate as
1838
+ # numeric 0/1; reject anything that doesn't fit a {0, 1} subset.
1839
+
1840
+ try:
1841
+ unique_y = pd.Series(plot_df[y].dropna().astype(float)).unique()
1842
+ except (ValueError, TypeError):
1843
+ raise ValueError(
1844
+ f"Binomial smoothing requires a numeric binary 0/1 outcome; "
1845
+ f"{y!r} could not be converted to numeric"
1846
+ ) from None
1847
+ if len(unique_y) != 2 or not set(unique_y).issubset({0.0, 1.0}):
1848
+ raise ValueError(
1849
+ f"Binomial smoothing requires a binary 0/1 outcome; {y!r} has "
1850
+ f"unique values: {sorted(unique_y)}"
1851
+ )
1852
+ if raw_data:
1853
+ p += geom_point(alpha=alpha_point)
1854
+ p = _add_binomial_smooth(p, data, x, y, uncertainty, level, bands, color=color)
1855
+ if overlay_specs:
1856
+ p = _add_overlay_binomial(p, data, x, y, overlay_specs)
1857
+
1858
+ elif not skip_dispatch and is_y_numeric and not is_x_discrete:
1859
+ if raw_data:
1860
+ p += geom_point(alpha=alpha_point)
1861
+ if interaction_model and variables.get("has_interaction") and color:
1862
+ # Non-parallel slopes per color group via statsmodels OLS with
1863
+ # the actual interaction term (e.g. y ~ x * color). v0.7.0+.
1864
+ p = _add_interaction_smooth(
1865
+ p, data, x, y, color, variables["all_x"],
1866
+ method, uncertainty, level, bands,
1867
+ )
1868
+ else:
1869
+ p = _add_numeric_smooth(
1870
+ p,
1871
+ data,
1872
+ x,
1873
+ y,
1874
+ method,
1875
+ uncertainty,
1876
+ level,
1877
+ bands,
1878
+ color=color,
1879
+ random_effects=random_effects,
1880
+ mixed_backend=mixed_backend,
1881
+ )
1882
+ if overlay_specs:
1883
+ p = _add_overlay_numeric(p, data, x, y, overlay_specs)
1884
+
1885
+ elif not skip_dispatch and is_y_numeric and is_x_discrete:
1886
+ if raw_data and jitter_xy[0] > 0:
1887
+ p += geom_jitter(width=jitter_xy[0], alpha=alpha_point)
1888
+ elif raw_data:
1889
+ p += geom_point(alpha=alpha_point)
1890
+ p = _add_discrete_summary(p, spread)
1891
+
1892
+ elif not skip_dispatch:
1893
+ if raw_data and jitter_xy != (0.0, 0.0):
1894
+ p += geom_jitter(width=jitter_xy[0], height=jitter_xy[1], alpha=alpha_point)
1895
+ elif raw_data:
1896
+ p += geom_point(alpha=alpha_point)
1897
+
1898
+ if len(given) == 1:
1899
+ p += facet_wrap(f"~{given[0]}")
1900
+ elif len(given) >= 2:
1901
+ p += facet_grid(f"{given[1]} ~ {given[0]}")
1902
+
1903
+ p += theme_bw()
1904
+
1905
+ # --- Optional plot.string override (v0.6.6+) ---
1906
+ # R-flexplot's plot.string is a dict of label overrides:
1907
+ # {"x": "Time (s)", "y": "Voltage (V)", "title": "Experiment 1"}
1908
+ # Validation: must be a dict with string keys and string values.
1909
+ if plot_string is not None:
1910
+ if not isinstance(plot_string, dict):
1911
+ raise TypeError(
1912
+ f"plot_string must be a dict of {{label: text}} overrides; "
1913
+ f"got {type(plot_string).__name__}."
1914
+ )
1915
+ bad = {k: type(v).__name__ for k, v in plot_string.items()
1916
+ if not isinstance(k, str) or not isinstance(v, str)}
1917
+ if bad:
1918
+ raise TypeError(
1919
+ f"plot_string keys and values must all be strings; "
1920
+ f"bad entries: {bad}"
1921
+ )
1922
+ # Apply via plotnine's labs(). Only known labels are passed through;
1923
+ # unknown keys are silently ignored (plotnine's labs() ignores
1924
+ # unknown keys but emits a warning we don't want to surface).
1925
+ labs_kwargs = {
1926
+ k: v for k, v in plot_string.items()
1927
+ if k in {"x", "y", "title", "subtitle", "caption", "color"}
1928
+ }
1929
+ if labs_kwargs:
1930
+ p += labs(**labs_kwargs)
1931
+
1932
+ # --- Optional ghost.line reference layer (v0.6.5+) ---
1933
+ # ghost_line="red": solid red reference line. Useful for highlighting a
1934
+ # threshold or a reference value (e.g. y=0, or y=mean(y)).
1935
+ # ghost_line="dashed": dashed black line. R's flexplot() uses this to
1936
+ # mark the slope=1 reference for prediction-vs-observed plots.
1937
+ # Both are drawn as geom_hline (horizontal), so they're 1D references
1938
+ # at y=0. For diagonal references (slope=1), future work.
1939
+ # --- ghost.line & ghost.reference (v0.8.0, R-parity) ---
1940
+ # R semantics: ghost.line is the COLOR of a line fit on a reference
1941
+ # panel that is repeated into every other panel (cross-panel
1942
+ # comparison). If ghost_reference is provided without an explicit
1943
+ # ghost_line, ghost_line defaults to "gray".
1944
+ has_grouping = len(given) >= 1 or color is not None
1945
+
1946
+ if ghost_reference is not None and not isinstance(ghost_reference, (dict, pd.DataFrame)):
1947
+ raise TypeError(
1948
+ f"ghost_reference must be None, a dict ({{given_var: level}}) for "
1949
+ f"panel reference selection, or a DataFrame for overlay; got "
1950
+ f"{type(ghost_reference).__name__}."
1951
+ )
1952
+
1953
+ if isinstance(ghost_reference, dict) and not has_grouping:
1954
+ raise TypeError(
1955
+ "ghost_reference dict requires a `| given` facet in the formula."
1956
+ )
1957
+
1958
+ if ghost_reference is not None and not isinstance(ghost_reference, pd.DataFrame) and ghost_line is None:
1959
+ ghost_line = "gray"
1960
+ if ghost_line is True:
1961
+ ghost_line = "gray"
1962
+
1963
+ if ghost_line is not None:
1964
+ if not isinstance(ghost_line, str):
1965
+ raise TypeError(
1966
+ f"ghost_line must be a color string, 'slope1', or None; "
1967
+ f"got {type(ghost_line).__name__}."
1968
+ )
1969
+ if has_grouping:
1970
+ # R-parity path: fit y ~ x on the reference subset and repeat
1971
+ # the predicted line into every panel/group, drawn in ghost_line's
1972
+ # color. Reference selection via ghost_reference dict
1973
+ # ({given_var: level}) or the first level of the first given
1974
+ # variable when absent.
1975
+ if x is None or is_x_discrete:
1976
+ # Ghost lines are only defined for numeric x fits.
1977
+ if not isinstance(ghost_line, str):
1978
+ raise TypeError("ghost_line must be a string.")
1979
+ ref_df: Optional[pd.DataFrame] = plot_input_df
1980
+ if ghost_reference is not None:
1981
+ if isinstance(ghost_reference, pd.DataFrame):
1982
+ # DataFrame overlay path (legacy) is handled AFTER this
1983
+ # block; dicts drive panel-referencing here.
1984
+ ref_df = None
1985
+ elif isinstance(ghost_reference, dict):
1986
+ for var, val in ghost_reference.items():
1987
+ if var not in plot_input_df.columns:
1988
+ raise ValueError(
1989
+ f"ghost_reference key {var!r} is not a "
1990
+ f"data column."
1991
+ )
1992
+ mask = (plot_input_df[var] == val) | (plot_input_df[var].astype(str) == str(val))
1993
+ if not mask.any():
1994
+ # Nearest-match fallback for numeric refs.
1995
+ if pd.api.types.is_numeric_dtype(plot_input_df[var]):
1996
+ idx = (plot_input_df[var] - val).abs().idxmin()
1997
+ mask = plot_input_df[var] == plot_input_df.loc[idx, var]
1998
+ else:
1999
+ mask = plot_input_df[var] == plot_input_df[var].iloc[0]
2000
+ ref_df = plot_input_df[mask]
2001
+ else:
2002
+ raise TypeError(
2003
+ "ghost_reference must be None, a dict "
2004
+ "({given_var: level}) for panel reference selection, "
2005
+ "or a DataFrame for overlay; got "
2006
+ f"{type(ghost_reference).__name__}."
2007
+ )
2008
+ if ref_df is not None and len(ref_df) > 1 and pd.api.types.is_numeric_dtype(ref_df[x]):
2009
+ gx = ref_df[x].to_numpy(dtype=float)
2010
+ gy = ref_df[y].to_numpy(dtype=float)
2011
+ valid = np.isfinite(gx) & np.isfinite(gy)
2012
+ gx_valid = gx[valid]
2013
+ gy_valid = gy[valid]
2014
+ if len(gx_valid) > 1:
2015
+ _X = np.column_stack([np.ones_like(gx_valid), gx_valid])
2016
+ _m = OLS(gy_valid, _X).fit()
2017
+ _x_eval = np.linspace(np.nanmin(gx_valid), np.nanmax(gx_valid), num=200)
2018
+ _y_eval = _m.predict(np.column_stack([np.ones_like(_x_eval), _x_eval]))
2019
+ ghost_df = pd.DataFrame({x: _x_eval, y: _y_eval})
2020
+ ghost_color = "black" if ghost_line == "dashed" else ghost_line
2021
+ p += geom_line(
2022
+ aes(x=x, y=y),
2023
+ data=ghost_df,
2024
+ color=ghost_color,
2025
+ linetype="dashed",
2026
+ inherit_aes=False,
2027
+ )
2028
+ else:
2029
+ # No grouping/facets: reference line at y=0 or diagonal slope1
2030
+ valid_unfaceted = {"red", "dashed", "slope1", "gray", "lightgray", "black", "blue", "darkgreen"}
2031
+ if ghost_line not in valid_unfaceted:
2032
+ raise ValueError(
2033
+ f"Without `| given` facets or color predictors, ghost_line must be 'gray', "
2034
+ f"'red', 'dashed', 'slope1', or None; got {ghost_line!r}. "
2035
+ f"Panel-repetition (R parity) requires a facet in the "
2036
+ f"formula."
2037
+ )
2038
+ if ghost_line == "slope1":
2039
+ from plotnine import geom_abline
2040
+ p += geom_abline(intercept=0, slope=1, color="black", linetype="dashed")
2041
+ else:
2042
+ ghost_color = "black" if ghost_line == "dashed" else ghost_line
2043
+ p += geom_hline(yintercept=0, color=ghost_color, linetype="dashed" if ghost_line == "dashed" else "solid")
2044
+
2045
+ # --- Optional ghost.reference overlay (v0.6.6+) ---
2046
+ # R-flexplot accepts ghost.reference as a DataFrame to overlay on the
2047
+ # same axes. Two common patterns:
2048
+ # 1. Reference scatter: columns matching x/y → draw geom_point in
2049
+ # light gray.
2050
+ # 2. Reference prediction line: columns (x, "pred") → draw geom_line
2051
+ # in a contrasting color.
2052
+ # We detect the pattern by checking if the DataFrame has columns
2053
+ # [x, y] or [x, "pred"].
2054
+ if ghost_reference is not None:
2055
+ # Dict form ({given_var: level}) was consumed by the ghost.line
2056
+ # panel-reference path above; only DataFrames reach this legacy
2057
+ # overlay path.
2058
+ if isinstance(ghost_reference, dict):
2059
+ # Dict = panel-reference selector, consumed by the ghost.line
2060
+ # block above. Here we only validate the facet requirement.
2061
+ if not has_grouping:
2062
+ raise TypeError(
2063
+ "ghost_reference dict requires a `| given` facet in the "
2064
+ "formula (it selects the reference panel)."
2065
+ )
2066
+ elif isinstance(ghost_reference, pd.DataFrame):
2067
+ pass # DataFrame overlay handled below.
2068
+ else:
2069
+ raise TypeError(
2070
+ f"ghost_reference must be None, a dict (panel reference), "
2071
+ f"or a pandas DataFrame (overlay); got "
2072
+ f"{type(ghost_reference).__name__}."
2073
+ )
2074
+ if isinstance(ghost_reference, pd.DataFrame) and x not in ghost_reference.columns:
2075
+ raise ValueError(
2076
+ f"ghost_reference DataFrame must have column {x!r} "
2077
+ f"(matching x in the formula); got columns "
2078
+ f"{list(ghost_reference.columns)}."
2079
+ )
2080
+ if isinstance(ghost_reference, dict):
2081
+ pass # dict refs don't carry overlay columns
2082
+ elif "pred" in ghost_reference.columns:
2083
+ # Prediction-line pattern: geom_line in red.
2084
+ p += geom_line(
2085
+ aes(x=x, y="pred"),
2086
+ data=ghost_reference,
2087
+ color="red",
2088
+ linetype="dashed",
2089
+ inherit_aes=False,
2090
+ )
2091
+ elif y in ghost_reference.columns:
2092
+ # Reference-scatter pattern: geom_point in light gray.
2093
+ p += geom_point(
2094
+ data=ghost_reference,
2095
+ color="gray",
2096
+ alpha=0.4,
2097
+ inherit_aes=False,
2098
+ )
2099
+ else:
2100
+ raise ValueError(
2101
+ f"ghost_reference must have either column {y!r} (scatter) "
2102
+ f"or 'pred' (line); got {list(ghost_reference.columns)}."
2103
+ )
2104
+
2105
+ if return_data:
2106
+ return {"plot": p, "data": plot_input_df}
2107
+ return p
2108
+
2109
+
2110
+ def _lowess_predict(x_eval: np.ndarray, y: np.ndarray, x: np.ndarray) -> np.ndarray:
2111
+ """Wrapper around statsmodels ``lowess`` that takes (x_eval, y_sorted_by_x)."""
2112
+ order = np.argsort(x)
2113
+ x_sorted = np.asarray(x)[order]
2114
+ y_sorted = np.asarray(y)[order]
2115
+ smoothed = lowess(y_sorted, x_sorted, return_sorted=False)
2116
+ return np.interp(x_eval, x_sorted, smoothed)
2117
+
2118
+
2119
+ def _add_numeric_smooth(
2120
+ p,
2121
+ data: pd.DataFrame,
2122
+ x: str,
2123
+ y: str,
2124
+ method: str,
2125
+ uncertainty: Optional[str],
2126
+ level: float,
2127
+ bands: Optional[List[float]],
2128
+ color: Optional[str] = None,
2129
+ random_effects: Optional[str] = None,
2130
+ mixed_backend: str = "auto",
2131
+ ):
2132
+ """Add fitted line + uncertainty band for numeric-vs-numeric.
2133
+
2134
+ Returns the plotnine plot object with the appropriate layers added.
2135
+ Caller is responsible for adding geom_point first.
2136
+ """
2137
+ if uncertainty is None:
2138
+ # No fit at all — preserve the scatter only.
2139
+ return p
2140
+
2141
+ line_color = None if color else "blue"
2142
+
2143
+ # polynomial/quadratic/cubic are OLS fits with higher-order x terms;
2144
+ # logistic/poisson/Gamma are GLMs; rlm is robust regression. plotnine's
2145
+ # geom_smooth does NOT support all of these cleanly, so we route them
2146
+ # through statsmodels and add geom_line + geom_ribbon manually.
2147
+ if method in {
2148
+ "quadratic",
2149
+ "polynomial",
2150
+ "cubic",
2151
+ "logistic",
2152
+ "rlm",
2153
+ "poisson",
2154
+ "Gamma",
2155
+ "mixedlm",
2156
+ "lmer",
2157
+ "glmer",
2158
+ }:
2159
+ return _add_parametric_smooth(
2160
+ p,
2161
+ data,
2162
+ x,
2163
+ y,
2164
+ method,
2165
+ uncertainty,
2166
+ level,
2167
+ bands,
2168
+ random_effects=random_effects,
2169
+ mixed_backend=mixed_backend,
2170
+ )
2171
+
2172
+ use_loess = method == "loess"
2173
+
2174
+ # --- Nested bands (multiple ribbons via multiple geom_smooth layers) ---
2175
+ if bands is not None:
2176
+ levels = sorted(set(bands))
2177
+ for lvl in levels:
2178
+ kwargs = {"method": "loess" if use_loess else "lm", "level": lvl, "alpha": 0.15}
2179
+ if line_color:
2180
+ kwargs["color"] = line_color
2181
+ p += cast(Any, geom_smooth)(**kwargs)
2182
+ return p
2183
+
2184
+ # --- Single band ---
2185
+ if uncertainty == "ci":
2186
+ kwargs = {"method": "loess" if use_loess else "lm", "level": level}
2187
+ if line_color:
2188
+ kwargs["color"] = line_color
2189
+ p += cast(Any, geom_smooth)(**kwargs)
2190
+ return p
2191
+
2192
+ if uncertainty == "prediction":
2193
+ # Fit an OLS model, compute residual-based PI on a sorted-x grid,
2194
+ # and draw a ribbon + the fitted line.
2195
+ from scipy import stats as _scipy_stats
2196
+
2197
+ x_arr = data[x].to_numpy(dtype=float)
2198
+ y_arr = data[y].to_numpy(dtype=float)
2199
+ model = OLS(y_arr, sm.add_constant(x_arr)).fit()
2200
+ x_eval = np.sort(np.unique(x_arr))
2201
+ if x_eval.size < 2:
2202
+ # Degenerate: cannot draw a meaningful band.
2203
+ p += geom_line(aes(y=y), color="blue")
2204
+ return p
2205
+ yhat_eval = model.predict(sm.add_constant(x_eval))
2206
+ yhat_full = model.predict(sm.add_constant(x_arr))
2207
+ sigma = float(np.sqrt(np.mean((y_arr - yhat_full) ** 2)))
2208
+ z = float(_scipy_stats.norm.ppf(0.5 + level / 2))
2209
+ half_width = z * sigma
2210
+ ribbon_df = pd.DataFrame({
2211
+ x: x_eval,
2212
+ "__lower": yhat_eval - half_width,
2213
+ "__upper": yhat_eval + half_width,
2214
+ y: yhat_eval,
2215
+ })
2216
+ p += geom_ribbon(
2217
+ aes(ymin="__lower", ymax="__upper"),
2218
+ data=ribbon_df,
2219
+ alpha=0.2,
2220
+ fill="blue",
2221
+ inherit_aes=False,
2222
+ )
2223
+ p += geom_line(
2224
+ aes(y=y),
2225
+ data=ribbon_df,
2226
+ color="blue",
2227
+ inherit_aes=False,
2228
+ )
2229
+ return p
2230
+
2231
+ if uncertainty == "bootstrap":
2232
+ # Case-resampled bootstrap CI for the loess branch.
2233
+ x_arr = data[x].to_numpy(dtype=float)
2234
+ y_arr = data[y].to_numpy(dtype=float)
2235
+ x_eval, lower, upper = compute_bootstrap_ci(
2236
+ x_arr, y_arr,
2237
+ smooth_fn=lambda x_e, y_s: _lowess_predict(x_e, y_s, x_arr),
2238
+ n_resamples=200,
2239
+ level=level,
2240
+ random_state=None,
2241
+ )
2242
+ smoothed_line = _lowess_predict(x_eval, y_arr, x_arr)
2243
+ ribbon_df = pd.DataFrame({
2244
+ x: x_eval,
2245
+ "__lower": lower,
2246
+ "__upper": upper,
2247
+ y: smoothed_line,
2248
+ })
2249
+ p += geom_ribbon(
2250
+ aes(ymin="__lower", ymax="__upper"),
2251
+ data=ribbon_df,
2252
+ alpha=0.2,
2253
+ fill="blue",
2254
+ inherit_aes=False,
2255
+ )
2256
+ p += geom_line(
2257
+ aes(y=y),
2258
+ data=ribbon_df,
2259
+ color="blue",
2260
+ inherit_aes=False,
2261
+ )
2262
+ return p
2263
+
2264
+ # Should never reach here thanks to validate_uncertainty_params.
2265
+ return p
2266
+
2267
+
2268
+ def _add_parametric_smooth(
2269
+ p,
2270
+ data: pd.DataFrame,
2271
+ x: str,
2272
+ y: str,
2273
+ method: str,
2274
+ uncertainty: Optional[str],
2275
+ level: float,
2276
+ bands: Optional[List[float]],
2277
+ random_effects: Optional[str] = None,
2278
+ mixed_backend: str = "auto",
2279
+ ):
2280
+ """Add fitted line + CI ribbon for polynomial / cubic / logistic methods.
2281
+
2282
+ plotnine's ``geom_smooth(method="lm", ...)`` does NOT accept
2283
+ ``formula=poly(x, k)`` cleanly, so we fit statsmodels directly and draw
2284
+ the line + ribbon manually. Mirrors the prediction/ bootstrap branches
2285
+ in ``_add_numeric_smooth``.
2286
+
2287
+ Methods:
2288
+ - "quadratic"/"polynomial": degree-2 OLS.
2289
+ - "cubic": degree-3 OLS.
2290
+ - "logistic": GLM with logit link on numeric binary y.
2291
+ - "mixedlm"/"lmer": linear mixed-effects with a random intercept.
2292
+ - "glmer": binomial mixed-effects with a random intercept.
2293
+ """
2294
+ from scipy import stats as _scipy_stats
2295
+
2296
+ def _parse_random_effects_spec(spec: Optional[str], x_name: str):
2297
+ if not spec:
2298
+ raise ValueError(
2299
+ "Mixed-effects methods require random_effects=. "
2300
+ "Use a group column name (e.g., random_effects='school') "
2301
+ "or an lme4-style mini spec (e.g., '(1|school)' or "
2302
+ "'(1 + x|school)')."
2303
+ )
2304
+ text = str(spec).strip()
2305
+ if text in data.columns:
2306
+ return "1", text
2307
+ m = re.match(r"^\(\s*(.+?)\s*\|\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)$", text)
2308
+ if not m:
2309
+ raise ValueError(
2310
+ f"Invalid random_effects specification {spec!r}. "
2311
+ "Expected a column name or '(effects|group)'."
2312
+ )
2313
+ re_formula_raw, group_col = m.group(1).strip(), m.group(2).strip()
2314
+ if group_col not in data.columns:
2315
+ raise ValueError(
2316
+ f"random_effects group column {group_col!r} not found in data."
2317
+ )
2318
+ if re_formula_raw == "1":
2319
+ return "1", group_col
2320
+ re_formula = re_formula_raw.replace(x_name, "x")
2321
+ allowed = {"1 + x", "x", "0 + x", "1+x", "0+x"}
2322
+ if re_formula not in allowed:
2323
+ raise NotImplementedError(
2324
+ "Supported random_effects formulas are '(1|g)' and "
2325
+ "'(1 + x|g)' (or '(x|g)')."
2326
+ )
2327
+ if re_formula in {"x", "0 + x", "0+x"}:
2328
+ re_formula = "0 + x"
2329
+ else:
2330
+ re_formula = "1 + x"
2331
+ return re_formula, group_col
2332
+
2333
+ def _add_fe_band(ribbon_df: pd.DataFrame, fe_mean: np.ndarray, fe_cov: np.ndarray):
2334
+ if bands is not None:
2335
+ levels = sorted(set(bands))
2336
+ else:
2337
+ levels = [level]
2338
+ for lvl in levels:
2339
+ z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
2340
+ se = np.sqrt(np.maximum(np.einsum("ij,jk,ik->i", fe_mean, fe_cov, fe_mean), 0.0))
2341
+ ribbon_df[f"__lower_{lvl}"] = ribbon_df[y].to_numpy() - z * se
2342
+ ribbon_df[f"__upper_{lvl}"] = ribbon_df[y].to_numpy() + z * se
2343
+ for lvl in sorted(levels, reverse=True):
2344
+ alpha = 0.1 + 0.15 * (lvl / max(levels))
2345
+ p_local = geom_ribbon(
2346
+ aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
2347
+ data=ribbon_df,
2348
+ alpha=alpha,
2349
+ fill="blue",
2350
+ inherit_aes=False,
2351
+ )
2352
+ yield p_local
2353
+
2354
+ x_arr = data[x].to_numpy(dtype=float)
2355
+ y_arr = data[y].to_numpy(dtype=float)
2356
+ n = x_arr.size
2357
+ if n < 2:
2358
+ return p
2359
+
2360
+ if method in {"quadratic", "polynomial"}:
2361
+ # R-flexplot: both "polynomial" and "quadratic" are degree-2 OLS.
2362
+ X = np.column_stack([np.ones_like(x_arr), x_arr, x_arr ** 2])
2363
+ model = OLS(y_arr, X).fit()
2364
+ link_label = "polynomial (degree-2)"
2365
+ elif method == "cubic":
2366
+ X = np.column_stack([np.ones_like(x_arr), x_arr, x_arr ** 2, x_arr ** 3])
2367
+ model = OLS(y_arr, X).fit()
2368
+ link_label = "cubic (degree-3)"
2369
+ elif method == "logistic":
2370
+ # Validate binary {0, 1}; fall back to OLS with a warning if not.
2371
+ unique_y = np.unique(y_arr[~np.isnan(y_arr)])
2372
+ is_binary = set(unique_y.tolist()).issubset({0.0, 1.0}) and len(unique_y) == 2
2373
+ if not is_binary:
2374
+ warnings.warn(
2375
+ f"method='logistic' requires a numeric binary 0/1 outcome; "
2376
+ f"{y!r} has unique values {sorted(unique_y.tolist())}. "
2377
+ f"Falling back to OLS.",
2378
+ UserWarning,
2379
+ stacklevel=3,
2380
+ )
2381
+ X = sm.add_constant(x_arr)
2382
+ model = OLS(y_arr, X).fit()
2383
+ link_label = "OLS fallback (logistic requires binary y)"
2384
+ else:
2385
+ X = sm.add_constant(x_arr)
2386
+ model = sm.GLM(
2387
+ y_arr, X, family=sm.families.Binomial(link=sm.families.links.Logit())
2388
+ ).fit()
2389
+ link_label = "logistic (logit)"
2390
+ elif method == "rlm":
2391
+ X = sm.add_constant(x_arr)
2392
+ model = sm.RLM(y_arr, X, M=sm.robust.norms.HuberT()).fit()
2393
+ link_label = "rlm (Huber)"
2394
+ elif method == "poisson":
2395
+ X = sm.add_constant(x_arr)
2396
+ if np.any(y_arr < 0):
2397
+ warnings.warn(
2398
+ f"method='poisson' requires a non-negative outcome; {y!r} has "
2399
+ f"negative values. Falling back to OLS.",
2400
+ UserWarning,
2401
+ stacklevel=3,
2402
+ )
2403
+ model = OLS(y_arr, X).fit()
2404
+ link_label = "OLS fallback (poisson requires non-negative y)"
2405
+ else:
2406
+ model = sm.GLM(
2407
+ y_arr, X, family=sm.families.Poisson(link=sm.families.links.Log())
2408
+ ).fit()
2409
+ link_label = "poisson (log)"
2410
+ elif method == "Gamma":
2411
+ X = sm.add_constant(x_arr)
2412
+ if np.any(y_arr <= 0):
2413
+ warnings.warn(
2414
+ f"method='Gamma' requires a strictly positive outcome; {y!r} has "
2415
+ f"non-positive values. Falling back to OLS.",
2416
+ UserWarning,
2417
+ stacklevel=3,
2418
+ )
2419
+ model = OLS(y_arr, X).fit()
2420
+ link_label = "OLS fallback (Gamma requires positive y)"
2421
+ else:
2422
+ model = sm.GLM(
2423
+ y_arr, X,
2424
+ family=sm.families.Gamma(link=sm.families.links.InversePower()),
2425
+ ).fit()
2426
+ link_label = "Gamma (inverse)"
2427
+ elif method in {"mixedlm", "lmer"}:
2428
+ if mixed_backend not in {"auto", "statsmodels"}:
2429
+ raise ValueError(
2430
+ f"mixed_backend must be 'auto' or 'statsmodels'; got {mixed_backend!r}."
2431
+ )
2432
+ re_formula, group_col = _parse_random_effects_spec(random_effects, x)
2433
+ fit_df = pd.DataFrame({"y": y_arr, "x": x_arr, "__group": data[group_col]})
2434
+ try:
2435
+ model = smf.mixedlm(
2436
+ "y ~ x",
2437
+ data=fit_df,
2438
+ groups=fit_df["__group"],
2439
+ re_formula=re_formula,
2440
+ ).fit(reml=True, method="lbfgs", disp=False)
2441
+ except Exception as exc:
2442
+ raise RuntimeError(
2443
+ f"Failed to fit mixedlm/lmer model: {exc}"
2444
+ ) from exc
2445
+ link_label = "mixedlm/lmer"
2446
+ elif method == "glmer":
2447
+ if mixed_backend not in {"auto", "statsmodels"}:
2448
+ raise ValueError(
2449
+ f"mixed_backend must be 'auto' or 'statsmodels'; got {mixed_backend!r}."
2450
+ )
2451
+ unique_y = np.unique(y_arr[~np.isnan(y_arr)])
2452
+ is_binary = set(unique_y.tolist()).issubset({0.0, 1.0}) and len(unique_y) == 2
2453
+ if not is_binary:
2454
+ raise ValueError(
2455
+ f"method='glmer' requires a numeric binary 0/1 outcome; "
2456
+ f"{y!r} has unique values {sorted(unique_y.tolist())}."
2457
+ )
2458
+ _, group_col = _parse_random_effects_spec(random_effects, x)
2459
+ fit_df = pd.DataFrame({"y": y_arr, "x": x_arr, "__group": data[group_col]})
2460
+ from statsmodels.genmod.bayes_mixed_glm import BinomialBayesMixedGLM
2461
+ try:
2462
+ model = BinomialBayesMixedGLM.from_formula(
2463
+ "y ~ x",
2464
+ {"__group": "0 + C(__group)"},
2465
+ fit_df,
2466
+ ).fit_vb()
2467
+ except Exception as exc:
2468
+ raise RuntimeError(f"Failed to fit glmer model: {exc}") from exc
2469
+ link_label = "glmer"
2470
+ else: # pragma: no cover — guarded by caller
2471
+ return p
2472
+
2473
+ x_eval = np.linspace(np.nanmin(x_arr), np.nanmax(x_arr), num=200)
2474
+
2475
+ # Build the corresponding evaluation design matrix.
2476
+ if method in {"quadratic", "polynomial"}:
2477
+ X_eval = np.column_stack([np.ones_like(x_eval), x_eval, x_eval ** 2])
2478
+ elif method == "cubic":
2479
+ X_eval = np.column_stack(
2480
+ [np.ones_like(x_eval), x_eval, x_eval ** 2, x_eval ** 3]
2481
+ )
2482
+ elif method in {"logistic", "rlm", "poisson", "Gamma"}:
2483
+ # logistic, rlm, poisson, Gamma (and logistic OLS fallback) are all
2484
+ # linear-in-x models: intercept + x.
2485
+ X_eval = sm.add_constant(x_eval)
2486
+ elif method in {"mixedlm", "lmer"}:
2487
+ X_eval = np.column_stack([np.ones_like(x_eval), x_eval])
2488
+ elif method == "glmer":
2489
+ X_eval = np.column_stack([np.ones_like(x_eval), x_eval])
2490
+ else: # pragma: no cover
2491
+ X_eval = sm.add_constant(x_eval)
2492
+
2493
+ if method in {"mixedlm", "lmer"}:
2494
+ fe_names = list(model.fe_params.index)
2495
+ fe_mean = np.asarray(model.fe_params.to_numpy(), dtype=float)
2496
+ cov_df = model.cov_params()
2497
+ if hasattr(cov_df, "loc"):
2498
+ fe_cov = np.asarray(cov_df.loc[fe_names, fe_names], dtype=float)
2499
+ else:
2500
+ fe_cov = np.asarray(cov_df, dtype=float)[: len(fe_names), : len(fe_names)]
2501
+ design = np.column_stack([np.ones_like(x_eval), x_eval])
2502
+ yhat_eval = design @ fe_mean
2503
+ ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
2504
+ for layer in _add_fe_band(ribbon_df, design, fe_cov):
2505
+ p += layer
2506
+ p += geom_line(aes(y=y), data=ribbon_df, color="blue", inherit_aes=False)
2507
+ _ = link_label
2508
+ return p
2509
+ if method == "glmer":
2510
+ beta = np.asarray(model.fe_mean, dtype=float)
2511
+ design = np.column_stack([np.ones_like(x_eval), x_eval])
2512
+ eta = design @ beta
2513
+ yhat_eval = 1.0 / (1.0 + np.exp(-eta))
2514
+ ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
2515
+ # Approximate FE-only bands from posterior SD; this ignores covariance,
2516
+ # but gives a stable uncertainty envelope without requiring MCMC draws.
2517
+ if bands is not None:
2518
+ levels = sorted(set(bands))
2519
+ else:
2520
+ levels = [level]
2521
+ fe_sd = np.asarray(getattr(model, "fe_sd", np.full_like(beta, np.nan)), dtype=float)
2522
+ se_eta = np.sqrt(np.maximum((design ** 2) @ (fe_sd ** 2), 0.0))
2523
+ for lvl in levels:
2524
+ z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
2525
+ lo = 1.0 / (1.0 + np.exp(-(eta - z * se_eta)))
2526
+ hi = 1.0 / (1.0 + np.exp(-(eta + z * se_eta)))
2527
+ ribbon_df[f"__lower_{lvl}"] = lo
2528
+ ribbon_df[f"__upper_{lvl}"] = hi
2529
+ for lvl in sorted(levels, reverse=True):
2530
+ alpha = 0.1 + 0.15 * (lvl / max(levels))
2531
+ p += geom_ribbon(
2532
+ aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
2533
+ data=ribbon_df,
2534
+ alpha=alpha,
2535
+ fill="blue",
2536
+ inherit_aes=False,
2537
+ )
2538
+ p += geom_line(aes(y=y), data=ribbon_df, color="blue", inherit_aes=False)
2539
+ _ = link_label
2540
+ return p
2541
+
2542
+ yhat_eval = np.asarray(model.predict(X_eval))
2543
+
2544
+ # --- Bands: nested or single ---
2545
+ if bands is not None:
2546
+ levels = sorted(set(bands))
2547
+ else:
2548
+ levels = [level]
2549
+
2550
+ # Outermost band draws the line; inner bands draw only the ribbon.
2551
+ # Build a single combined ribbon dataframe with all band columns so we
2552
+ # can layer them on the same plot. Outermost band = widest.
2553
+ ribbon_df = pd.DataFrame({x: x_eval, y: yhat_eval})
2554
+ for lvl in levels:
2555
+ z = float(_scipy_stats.norm.ppf(0.5 + lvl / 2))
2556
+ # Use the prediction SE for the mean (not for new observations) for
2557
+ # a CI-style band. statsmodels' get_prediction().summary_frame(alpha)
2558
+ # gives mean_ci_lower / mean_ci_upper directly.
2559
+ try:
2560
+ pred = model.get_prediction(X_eval)
2561
+ frame = pred.summary_frame(alpha=1 - lvl)
2562
+ lower = frame["mean_ci_lower"].to_numpy()
2563
+ upper = frame["mean_ci_upper"].to_numpy()
2564
+ except Exception:
2565
+ # Fallback: normal-approx CI using the model's parameter covariance.
2566
+ # For RLM, get_prediction() is not available, so we compute
2567
+ # Var(x' beta) = diag(X_eval @ Cov(beta) @ X_eval').
2568
+ cov_params = getattr(model, "cov_params", None)
2569
+ if cov_params is not None:
2570
+ try:
2571
+ var = np.einsum("ij,jk,ik->i", X_eval, cov_params(), X_eval)
2572
+ se = np.sqrt(var)
2573
+ except Exception:
2574
+ se = np.full(len(yhat_eval), np.nan)
2575
+ else:
2576
+ se = np.full(len(yhat_eval), np.nan)
2577
+ # Guard degenerate SE (e.g., perfect fit); draw a flat band.
2578
+ finite_se = np.where(np.isfinite(se), se, 0.0)
2579
+ lower = yhat_eval - z * finite_se
2580
+ upper = yhat_eval + z * finite_se
2581
+
2582
+ ribbon_df[f"__lower_{lvl}"] = lower
2583
+ ribbon_df[f"__upper_{lvl}"] = upper
2584
+
2585
+ # Draw ribbons (innermost first so outermost ends up on top).
2586
+ for lvl in sorted(levels, reverse=True):
2587
+ alpha = 0.1 + 0.15 * (lvl / max(levels))
2588
+ p += geom_ribbon(
2589
+ aes(ymin=f"__lower_{lvl}", ymax=f"__upper_{lvl}"),
2590
+ data=ribbon_df,
2591
+ alpha=alpha,
2592
+ fill="blue",
2593
+ inherit_aes=False,
2594
+ )
2595
+
2596
+ p += geom_line(
2597
+ aes(y=y),
2598
+ data=ribbon_df,
2599
+ color="blue",
2600
+ inherit_aes=False,
2601
+ )
2602
+
2603
+ # Inject the link label into the plot's labels so users can see what
2604
+ # was fit. plotnine exposes .labels; mutate via a workaround (geom_line
2605
+ # doesn't carry labels, so attach as a one-off text annotation is
2606
+ # cleaner — but text annotations need positioning data. Skipping for
2607
+ # now; users can pass `labs()` themselves).
2608
+ _ = link_label # reserved for future annotation hook
2609
+ return p
2610
+
2611
+
2612
+ def _add_interaction_smooth(
2613
+ p,
2614
+ data: pd.DataFrame,
2615
+ x: str,
2616
+ y: str,
2617
+ color: str,
2618
+ all_x: list,
2619
+ method: str,
2620
+ uncertainty: Optional[str],
2621
+ level: float,
2622
+ bands: Optional[List[float]],
2623
+ ):
2624
+ """Add per-color-group fitted lines for an interaction formula.
2625
+
2626
+ Used when ``interaction_model=True`` and the formula contains ``*`` or
2627
+ ``:`` syntax. Fits a statsmodels OLS with the actual interaction term
2628
+ (e.g. ``y ~ x * z`` rather than ``y ~ x + z``) and overlays one
2629
+ ``geom_line`` + optional ``geom_ribbon`` per level of ``color``.
2630
+
2631
+ Parameters
2632
+ ----------
2633
+ all_x : list of str
2634
+ The expanded predictor list from the parser; for ``y ~ x*z`` this
2635
+ is ``['x', 'z', 'x:z']``. The interaction term is auto-detected as
2636
+ any term containing ``:``.
2637
+ """
2638
+ # Identify the interaction term in all_x (the one containing ":").
2639
+ interaction_term = next((t for t in all_x if ":" in t), None)
2640
+ if interaction_term is None:
2641
+ # Defensive: if interaction_model=True but no interaction term
2642
+ # was parsed, fall back to the additive path. (Should not happen
2643
+ # if the parser is consistent, but we don't want to crash.)
2644
+ return _add_numeric_smooth(
2645
+ p, data, x, y, method, uncertainty, level, bands
2646
+ )
2647
+
2648
+ # Find the color column name in the interaction term: "x:z" -> first atom.
2649
+ color_atom = _first_atom(interaction_term.split(":")[1]) if ":" in interaction_term else color
2650
+ if color_atom != color:
2651
+ # Mismatch — fallback.
2652
+ return _add_numeric_smooth(
2653
+ p, data, x, y, method, uncertainty, level, bands
2654
+ )
2655
+
2656
+ # Build the design matrix: y ~ x + color + x:color (the interaction).
2657
+ x_arr = data[x].to_numpy(dtype=float)
2658
+ color_arr = data[color].to_numpy()
2659
+ y_arr = data[y].to_numpy(dtype=float)
2660
+
2661
+ # Encode color via a category code so the interaction term is numeric.
2662
+ color_series = pd.Series(color_arr)
2663
+ color_codes, color_levels = pd.factorize(color_series)
2664
+ color_codes = color_codes.astype(float)
2665
+ n_groups = len(color_levels)
2666
+ if n_groups < 2:
2667
+ # Degenerate: only one color level. Fall back to additive.
2668
+ return _add_numeric_smooth(
2669
+ p, data, x, y, method, uncertainty, level, bands
2670
+ )
2671
+
2672
+ # Build design matrix: intercept + x + color_dummies (drop first) + x:color_dummies
2673
+ # Simpler approach: use statsmodels' formula API directly with the
2674
+ # interaction term. This is cleaner than constructing the design
2675
+ # matrix by hand and matches R's `y ~ x * color` semantics.
2676
+ import statsmodels.formula.api as _smf
2677
+
2678
+ # Build a temporary DataFrame for statsmodels.
2679
+ fit_df = pd.DataFrame({
2680
+ "_y": y_arr,
2681
+ "_x": x_arr,
2682
+ "_color": color_arr,
2683
+ })
2684
+ # Renaming so statsmodels' patsy accepts them (avoid operator parsing issues).
2685
+ fit_df.columns = ["_y", "_x", "_color"]
2686
+ formula_str = "_y ~ _x * _color"
2687
+ try:
2688
+ model = _smf.ols(formula_str, data=fit_df).fit()
2689
+ except Exception as exc:
2690
+ raise RuntimeError(
2691
+ f"interaction_model=True requires a valid OLS fit with the "
2692
+ f"interaction term; got: {exc}"
2693
+ ) from exc
2694
+
2695
+ # Predict on a per-color grid.
2696
+ x_min = float(np.nanmin(x_arr))
2697
+ x_max = float(np.nanmax(x_arr))
2698
+ x_eval = np.linspace(x_min, x_max, num=200)
2699
+
2700
+ # Build eval DataFrame with one row per (x_eval, color_level).
2701
+ eval_rows = []
2702
+ for level_val in color_levels:
2703
+ for xv in x_eval:
2704
+ eval_rows.append({"_x": xv, "_color": level_val})
2705
+ eval_df = pd.DataFrame(eval_rows)
2706
+ yhat_eval = np.asarray(model.predict(eval_df))
2707
+
2708
+ # Compute CI bands. Supports a single level (level=) or nested bands
2709
+ # (bands=[...]). Returns a dict {level_value: (lower_array, upper_array)}
2710
+ # or None if CI computation failed / is suppressed.
2711
+ band_arrays: Optional[dict[float, tuple[Any, Any]]] = None
2712
+ if uncertainty in {"ci", "prediction"}:
2713
+ levels_to_compute = sorted(set(bands)) if bands is not None else [level]
2714
+ band_arrays = {}
2715
+ ci_kind = "obs_ci" if uncertainty == "prediction" else "mean_ci"
2716
+ for lvl in levels_to_compute:
2717
+ try:
2718
+ pred = model.get_prediction(eval_df)
2719
+ frame = pred.summary_frame(alpha=1 - lvl)
2720
+ lower = frame[f"{ci_kind}_lower"].to_numpy()
2721
+ upper = frame[f"{ci_kind}_upper"].to_numpy()
2722
+ band_arrays[lvl] = (lower, upper)
2723
+ except Exception:
2724
+ # Skip this level if statsmodels can't produce it.
2725
+ pass
2726
+ if not band_arrays:
2727
+ band_arrays = None
2728
+
2729
+ # Determine colors per group using a default palette.
2730
+ palette = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"]
2731
+ for i, level_val in enumerate(color_levels):
2732
+ group_mask = (eval_df["_color"] == level_val).to_numpy()
2733
+ line_color = palette[i % len(palette)]
2734
+ # Build a per-group dataframe for plotnine.
2735
+ line_df = pd.DataFrame({
2736
+ x: x_eval,
2737
+ y: yhat_eval[group_mask],
2738
+ color: np.full_like(x_eval, level_val, dtype=object),
2739
+ })
2740
+ # The line itself.
2741
+ p += geom_line(
2742
+ data=line_df,
2743
+ color=line_color,
2744
+ inherit_aes=False,
2745
+ )
2746
+ # Optional ribbons. For nested bands, draw innermost first so the
2747
+ # outermost band ends up on top.
2748
+ if band_arrays is not None:
2749
+ for lvl, (ci_lower, ci_upper) in sorted(band_arrays.items()):
2750
+ ribbon_df = line_df.copy()
2751
+ ribbon_df["__lower"] = ci_lower[group_mask]
2752
+ ribbon_df["__upper"] = ci_upper[group_mask]
2753
+ # Outer (larger coverage) bands are wider; lower alpha for
2754
+ # the innermost so the layering reads as Tufte-style nested.
2755
+ alpha = 0.10 + 0.10 * (lvl / max(band_arrays.keys()))
2756
+ p += geom_ribbon(
2757
+ aes(ymin="__lower", ymax="__upper"),
2758
+ data=ribbon_df,
2759
+ alpha=alpha,
2760
+ fill=line_color,
2761
+ inherit_aes=False,
2762
+ )
2763
+ return p
2764
+
2765
+
2766
+ def _add_binomial_smooth(
2767
+ p,
2768
+ data: pd.DataFrame,
2769
+ x: str,
2770
+ y: str,
2771
+ uncertainty: Optional[str],
2772
+ level: float,
2773
+ bands: Optional[List[float]],
2774
+ color: Optional[str] = None,
2775
+ ):
2776
+ """Add fitted line + uncertainty band for numeric-vs-binary (binomial GLM).
2777
+
2778
+ The band is always rendered on the response (probability) scale.
2779
+ """
2780
+ if uncertainty is None:
2781
+ # No fit at all — just the scatter.
2782
+ return p
2783
+
2784
+ method_args = {"family": "binomial"}
2785
+ line_color = None if color else "blue"
2786
+
2787
+ if bands is not None:
2788
+ levels = sorted(set(bands))
2789
+ for lvl in levels:
2790
+ kwargs = {
2791
+ "method": "glm",
2792
+ "method_args": method_args,
2793
+ "level": lvl,
2794
+ "alpha": 0.15,
2795
+ }
2796
+ if line_color:
2797
+ kwargs["color"] = line_color
2798
+ p += cast(Any, geom_smooth)(**kwargs)
2799
+ return p
2800
+
2801
+ kwargs = {
2802
+ "method": "glm",
2803
+ "method_args": method_args,
2804
+ "level": level,
2805
+ }
2806
+ if line_color:
2807
+ kwargs["color"] = line_color
2808
+ p += cast(Any, geom_smooth)(**kwargs)
2809
+ return p
2810
+
2811
+
2812
+ def _add_overlay_numeric(p, data, x, y, overlay_specs):
2813
+ """Add one geom_smooth per overlay spec on the numeric-vs-numeric branch.
2814
+
2815
+ Each entry is drawn with its own color and ``uncertainty``/``level``
2816
+ settings. No bootstrap overlay for non-loess methods (plotnine doesn't
2817
+ expose stat_smooth's bootstrap from a single ``geom_smooth`` call with
2818
+ arbitrary methods; we keep the API consistent by routing all overlays
2819
+ through ``geom_smooth`` and reserving bootstrap for ``"loess"``).
2820
+ """
2821
+ has_labels = any(
2822
+ spec.get("label") and spec["label"] != spec["method"]
2823
+ for spec in overlay_specs
2824
+ )
2825
+ label_colors = {}
2826
+
2827
+ for spec in overlay_specs:
2828
+ method = spec["method"]
2829
+ level = spec["level"]
2830
+ color = spec["color"]
2831
+ label = spec.get("label", method)
2832
+ kwargs = {"method": method, "level": level, "color": color}
2833
+ # Forward any extra stat_args (span, formula, method_args, ...) that
2834
+ # the user provided.
2835
+ for k in ("span", "formula", "method_args", "n"):
2836
+ if k in spec:
2837
+ kwargs[k] = spec[k]
2838
+ p += cast(Any, geom_smooth)(**kwargs)
2839
+ if has_labels:
2840
+ label_colors[label] = color
2841
+
2842
+ if has_labels and label_colors:
2843
+ # Add a manual color scale so labels appear in the legend.
2844
+ p += scale_color_manual(
2845
+ name="Method",
2846
+ values=label_colors,
2847
+ )
2848
+ return p
2849
+
2850
+
2851
+ def _add_overlay_binomial(p, data, x, y, overlay_specs):
2852
+ """Add binomial GLM overlay smoothers on the numeric-vs-binary branch.
2853
+
2854
+ Only entries with ``method == "glm"`` are supported here; other methods
2855
+ raise so the user gets a clear error rather than a silently-broken chart.
2856
+ """
2857
+ label_colors = {}
2858
+ for spec in overlay_specs:
2859
+ if spec["method"] != "glm":
2860
+ raise ValueError(
2861
+ f"Overlay method {spec['method']!r} is not supported on the "
2862
+ f"binomial branch; only 'glm' is allowed."
2863
+ )
2864
+ kwargs = {
2865
+ "method": "glm",
2866
+ "method_args": {"family": "binomial"},
2867
+ "level": spec["level"],
2868
+ "color": spec["color"],
2869
+ }
2870
+ p += cast(Any, geom_smooth)(**kwargs)
2871
+ label_colors[spec.get("label", "glm")] = spec["color"]
2872
+ if label_colors:
2873
+ p += scale_color_manual(name="Method", values=label_colors)
2874
+ return p
2875
+
2876
+
2877
+ def _first_non_intercept_name(model, fallback="x"):
2878
+ """Return the first non-intercept exog name or None if only intercept."""
2879
+ names = getattr(getattr(model, "model", None), "exog_names", None) or []
2880
+ non_intercept = [n for n in names if n not in ("Intercept", "const")]
2881
+ return non_intercept[0] if non_intercept else None
2882
+
2883
+
2884
+ def _is_neural_net_fit(model) -> bool:
2885
+ """Duck-type detection of a :class:`pyflexplot.flex_nn.NeuralNetFit`.
2886
+
2887
+ Avoids importing :mod:`flex_nn` at module load time so the core module
2888
+ stays cheap to import when neural-net support isn't needed. A
2889
+ ``NeuralNetFit`` exposes ``.predict(data)`` returning an indexed
2890
+ Series plus the response-var metadata the wrapper carries.
2891
+ """
2892
+ cls = type(model)
2893
+ cls_name = f"{cls.__module__}.{cls.__qualname__}"
2894
+ return cls_name == "pyflexplot.flex_nn.NeuralNetFit"
2895
+
2896
+
2897
+ def _visualize_neural_net(fit, data=None, **kwargs):
2898
+ """Visualization path for ``NeuralNetFit`` wrappers.
2899
+
2900
+ Mirrors the statsmodels ``visualize()`` output (predicted-vs-actual
2901
+ line on top of a scatter) so a user can drop a fitted network into
2902
+ the same plot they would have used for an OLS fit.
2903
+ """
2904
+ if data is None:
2905
+ raise ValueError(
2906
+ "visualize() requires data= when called on a NeuralNetFit "
2907
+ "(the wrapper does not carry the training data)."
2908
+ )
2909
+ if not isinstance(data, pd.DataFrame):
2910
+ raise TypeError(
2911
+ f"data must be a pandas DataFrame, got {type(data).__name__}"
2912
+ )
2913
+ if data.empty:
2914
+ raise ValueError("data must be non-empty for visualization.")
2915
+
2916
+ response_var = fit.response_var
2917
+ if response_var not in data.columns:
2918
+ raise ValueError(
2919
+ f"Response column {response_var!r} (declared on the NeuralNetFit) "
2920
+ f"not found in data. Available columns: {list(data.columns)}"
2921
+ )
2922
+
2923
+ # Determine the x predictor. Honour explicit x=, otherwise use the
2924
+ # first declared predictor (mirrors the statsmodels fallback in
2925
+ # visualize()).
2926
+ x_name = kwargs.get("x")
2927
+ if x_name is None:
2928
+ if not fit.predictor_names:
2929
+ raise ValueError(
2930
+ "NeuralNetFit has no declared predictor_names; "
2931
+ "pass an explicit x= argument."
2932
+ )
2933
+ x_name = fit.predictor_names[0]
2934
+ if x_name not in data.columns:
2935
+ raise ValueError(
2936
+ f"Predictor column {x_name!r} not found in data for visualization."
2937
+ )
2938
+
2939
+ pred = fit.predict(data)
2940
+
2941
+ plot_df = data.copy()
2942
+ plot_df["__predicted"] = pred.reindex(plot_df.index)
2943
+
2944
+ p = (
2945
+ ggplot(plot_df, aes(x=x_name, y=response_var))
2946
+ + geom_point(alpha=0.4)
2947
+ + geom_line(aes(y="__predicted"), color="red", size=1)
2948
+ + labs(
2949
+ title=f"Visualization: NeuralNetFit ({fit.backend})",
2950
+ subtitle=f"Predicted {response_var} vs {x_name}",
2951
+ )
2952
+ + theme_bw()
2953
+ )
2954
+ return p
2955
+
2956
+
2957
+ def visualize(
2958
+ model,
2959
+ data: Optional[pd.DataFrame] = None,
2960
+ plot: str = "model",
2961
+ **kwargs,
2962
+ ):
2963
+ """
2964
+ Provides a visual representation of a fitted statistical object.
2965
+ Supports statsmodels (OLS, GLM), scikit-learn models, and
2966
+ :class:`pyflexplot.flex_nn.NeuralNetFit` wrappers.
2967
+
2968
+ Parameters
2969
+ ----------
2970
+ model : fitted model
2971
+ Any model with a ``predict`` method. Statsmodels (OLS, GLM),
2972
+ scikit-learn regressors, and ``NeuralNetFit`` are supported.
2973
+ data : pd.DataFrame, optional
2974
+ Predictor data. If omitted, inferred from ``model.model.data``.
2975
+ plot : {"model", "residuals", "all"}, default "model"
2976
+ What to draw:
2977
+ - ``"model"``: predicted-vs-observed scatter with the fitted
2978
+ line (the legacy behavior).
2979
+ - ``"residuals"``: residual-vs-fitted scatter and a residual
2980
+ histogram, side by side. Mirrors R's
2981
+ ``visualize.lm(plot="residuals")``.
2982
+ - ``"all"``: a combined ``cowplot``-style panel with the model
2983
+ fit on the left and the residual plots on the right.
2984
+ When ``cowplot`` isn't installed, the components are returned
2985
+ as a dict of named layers.
2986
+
2987
+ Returns
2988
+ -------
2989
+ plotnine.ggplot OR dict
2990
+ When ``plot="model"`` or ``plot="all"`` (with cowplot), a
2991
+ single ggplot (or cowplot-joined object). When
2992
+ ``plot="residuals"``, a dict with ``"rvf"`` (residual-vs-fitted)
2993
+ and ``"hist"`` (residual histogram) ggplot objects.
2994
+ """
2995
+ # NeuralNetFit path: duck-typed dispatch so core.py doesn't have to
2996
+ # import flex_nn at module load time.
2997
+ if _is_neural_net_fit(model):
2998
+ return _visualize_neural_net(model, data=data, **kwargs)
2999
+
3000
+ plot = str(plot).lower()
3001
+ valid_plots = ("model", "residuals", "all")
3002
+ if plot not in valid_plots:
3003
+ raise ValueError(
3004
+ f"plot must be one of {valid_plots}; got {plot!r}."
3005
+ )
3006
+
3007
+ if data is None:
3008
+ if hasattr(model, "model") and hasattr(model.model, "data"):
3009
+ data = pd.DataFrame(model.model.data.orig_endog).join(
3010
+ pd.DataFrame(model.model.data.orig_exog)
3011
+ )
3012
+ else:
3013
+ raise ValueError("No data provided for visualization.")
3014
+
3015
+ if not isinstance(data, pd.DataFrame):
3016
+ raise TypeError(f"data must be a pandas DataFrame, got {type(data).__name__}")
3017
+ if data.empty:
3018
+ raise ValueError("data must be non-empty for visualization.")
3019
+
3020
+ if not hasattr(model, "predict"):
3021
+ raise NotImplementedError(
3022
+ f"Visualization for {type(model).__name__} not yet implemented "
3023
+ "(model has no predict method)."
3024
+ )
3025
+
3026
+ endog_names = getattr(getattr(model, "model", None), "endog_names", None)
3027
+ if endog_names is None:
3028
+ raise ValueError("Cannot determine outcome variable name from model.")
3029
+
3030
+ # Statsmodels predict returns a Series aligned to the input index.
3031
+ y_pred = model.predict(data)
3032
+ if not hasattr(y_pred, "index"):
3033
+ y_pred = pd.Series(y_pred, index=data.index)
3034
+
3035
+ plot_df = data.copy()
3036
+ plot_df["__predicted"] = y_pred.reindex(plot_df.index)
3037
+ if endog_names in plot_df.columns:
3038
+ plot_df["__residual"] = plot_df[endog_names] - plot_df["__predicted"]
3039
+ else:
3040
+ plot_df["__residual"] = pd.Series(
3041
+ getattr(model, "resid", pd.Series(dtype=float))
3042
+ ).reindex(plot_df.index)
3043
+
3044
+ # Determine the first predictor robustly.
3045
+ x_name = kwargs.get("x")
3046
+ if x_name is None:
3047
+ x_name = _first_non_intercept_name(model)
3048
+ if x_name is None:
3049
+ raise ValueError(
3050
+ "visualize() requires a non-intercept model or explicit x= argument."
3051
+ )
3052
+ if x_name not in plot_df.columns:
3053
+ raise ValueError(
3054
+ f"Predictor column {x_name!r} not found in data for visualization."
3055
+ )
3056
+
3057
+ # The "model" panel — legacy predicted-vs-observed plot.
3058
+ p_model = (
3059
+ ggplot(plot_df, aes(x=x_name, y=endog_names))
3060
+ + geom_point(alpha=0.4)
3061
+ + geom_line(aes(y="__predicted"), color="red", size=1)
3062
+ + labs(
3063
+ title=f"Visualization: {type(model).__name__}",
3064
+ subtitle=f"Predicted {endog_names} vs {x_name}",
3065
+ )
3066
+ + theme_bw()
3067
+ )
3068
+
3069
+ if plot == "model":
3070
+ return p_model
3071
+
3072
+ # Residual plots (used for plot="residuals" or plot="all").
3073
+ plot_resid_df = plot_df.dropna(subset=["__residual", "__predicted"])
3074
+ p_rvf = (
3075
+ ggplot(plot_resid_df, aes(x="__predicted", y="__residual"))
3076
+ + geom_point(alpha=0.4)
3077
+ + geom_hline(yintercept=0, linetype="dashed", color="gray")
3078
+ + labs(
3079
+ title="Residuals vs Predicted",
3080
+ x="Predicted",
3081
+ y="Residual",
3082
+ )
3083
+ + theme_bw()
3084
+ )
3085
+ p_hist = (
3086
+ ggplot(plot_resid_df, aes(x="__residual"))
3087
+ + geom_histogram(bins=30, fill="steelblue", color="white")
3088
+ + labs(
3089
+ title="Residual Distribution",
3090
+ x="Residual",
3091
+ y="Count",
3092
+ )
3093
+ + theme_bw()
3094
+ )
3095
+
3096
+ if plot == "residuals":
3097
+ return {"rvf": p_rvf, "hist": p_hist}
3098
+
3099
+ # plot == "all" — try to compose with cowplot; otherwise return dict.
3100
+ try:
3101
+ import cowplot # type: ignore
3102
+ return cowplot.plot_grid(
3103
+ p_model, p_rvf, p_hist,
3104
+ ncol=2,
3105
+ rel_widths=(0.6, 0.4),
3106
+ )
3107
+ except ImportError:
3108
+ return {
3109
+ "model": p_model,
3110
+ "rvf": p_rvf,
3111
+ "hist": p_hist,
3112
+ }
3113
+
3114
+
3115
+ def compare_fits(
3116
+ formula: str,
3117
+ data: pd.DataFrame,
3118
+ model1,
3119
+ model2,
3120
+ labels: List[str] = ["Model 1", "Model 2"],
3121
+ return_preds: bool = False,
3122
+ pred_type: str = "response",
3123
+ report_se: bool = False,
3124
+ re: bool = False,
3125
+ num_points: Optional[int] = None,
3126
+ clusters: Optional[int] = None,
3127
+ **kwargs,
3128
+ ):
3129
+ """
3130
+ Visually compare the fit of two different models (statsmodels/sklearn).
3131
+
3132
+ Parameters
3133
+ ----------
3134
+ formula, data, model1, model2, labels :
3135
+ Existing arguments; see prior releases.
3136
+ return_preds : bool, default False
3137
+ When True, return the prediction DataFrame instead of the plot.
3138
+ Columns: the formula's predictor(s), ``__y`` (the observed
3139
+ response), and ``__pred1`` / ``__pred2`` (each model's
3140
+ prediction). This matches R's ``compare.fits(..., return.preds=TRUE)``.
3141
+ pred_type : {"response", "link"}, default "response"
3142
+ Type of predictions passed to ``statsmodels`` for GLM models.
3143
+ ``"response"`` returns the probability scale (default);
3144
+ ``"link"`` returns the linear-predictor scale. Ignored for
3145
+ non-GLM models. Matches R's ``compare.fits(..., pred.type=...)``.
3146
+ report_se : bool, default False
3147
+ R-parity stub: when True, would include standard-error bands on
3148
+ prediction lines. Currently ignored; reserved for a future release.
3149
+ re : bool, default False
3150
+ R-parity stub: when True, would include random-effects predictions
3151
+ for mixed models. Currently ignored; reserved for a future release.
3152
+ num_points : int, optional
3153
+ R-parity stub: when set, would evaluate predictions on a grid of
3154
+ ``num_points`` points spanning the x-range. Currently ignored;
3155
+ reserved for a future release.
3156
+ clusters : int, optional
3157
+ R-parity stub: when set, would cluster the prediction grid.
3158
+ Currently ignored; reserved for a future release.
3159
+
3160
+ Returns
3161
+ -------
3162
+ plotnine.ggplot OR pandas.DataFrame
3163
+ When ``return_preds=False``, the comparison plot. When
3164
+ ``return_preds=True``, a DataFrame with observed and predicted
3165
+ values for both models.
3166
+ """
3167
+ if report_se or re or num_points is not None or clusters is not None:
3168
+ warnings.warn(
3169
+ "compare_fits(): arguments report_se/re/num_points/clusters are "
3170
+ "accepted for R API parity but are currently no-ops in py-flexplot.",
3171
+ UserWarning,
3172
+ stacklevel=2,
3173
+ )
3174
+
3175
+ variables = parse_flexplot_formula(formula)
3176
+ _validate_data_for_plot(formula, data, variables)
3177
+
3178
+ y_name = variables["y"]
3179
+ x_name = variables["x"]
3180
+
3181
+ pred1 = _get_model_predictions(model1, data, pred_type=pred_type)
3182
+ pred2 = _get_model_predictions(model2, data, pred_type=pred_type)
3183
+
3184
+ if len(pred1) != len(data) or len(pred2) != len(data):
3185
+ raise ValueError(
3186
+ f"Model predictions must match data length ({len(data)}): "
3187
+ f"got {len(pred1)} and {len(pred2)}."
3188
+ )
3189
+
3190
+ plot_df = data.copy()
3191
+ plot_df["__m1"] = pred1
3192
+ plot_df["__m2"] = pred2
3193
+
3194
+ if return_preds:
3195
+ return plot_df
3196
+
3197
+ p = (
3198
+ ggplot(plot_df, aes(x=x_name, y=y_name))
3199
+ + geom_point(alpha=0.3)
3200
+ + geom_line(aes(y="__m1", color='"#3498db"'), size=1)
3201
+ + geom_line(aes(y="__m2", color='"#e74c3c"'), size=1)
3202
+ + scale_color_identity(
3203
+ guide="legend",
3204
+ name="Model",
3205
+ labels=labels,
3206
+ breaks=["#3498db", "#e74c3c"],
3207
+ )
3208
+ + labs(title="Visual Model Comparison", x=x_name, y=y_name)
3209
+ + theme_bw()
3210
+ )
3211
+
3212
+ return p
3213
+
3214
+
3215
+ def third_eye(*args, **kwargs):
3216
+ """R API placeholder for ``third.eye``.
3217
+
3218
+ The R package exposes ``third.eye`` as a specialized 3-way interaction
3219
+ visualization. py-flexplot intentionally keeps this as a stub for now,
3220
+ so API discovery/parity tooling can detect the endpoint while behavior
3221
+ remains explicitly unimplemented.
3222
+ """
3223
+ raise NotImplementedError(
3224
+ "third_eye() is not implemented in py-flexplot yet. "
3225
+ "Use flexplot(..., interaction_model=True) for interaction visuals."
3226
+ )
3227
+
3228
+
3229
+ def _get_model_predictions(
3230
+ model,
3231
+ data: pd.DataFrame,
3232
+ pred_type: str = "response",
3233
+ ) -> pd.Series:
3234
+ """Return a pandas Series of predictions aligned to data.index.
3235
+
3236
+ Parameters
3237
+ ----------
3238
+ model : fitted model with ``predict`` method
3239
+ data : pd.DataFrame
3240
+ Predictor data.
3241
+ pred_type : {"response", "link"}, default "response"
3242
+ Passed through to ``statsmodels`` GLM ``predict``; controls
3243
+ whether the prediction is on the response or link scale.
3244
+ Ignored for non-GLM models.
3245
+ """
3246
+ if hasattr(model, "predict"):
3247
+ try:
3248
+ # Statsmodels fitted models accept a DataFrame and return a Series
3249
+ # indexed by the (possibly reduced) observation index. GLM models
3250
+ # accept a ``linear`` kwarg to switch between response and link.
3251
+ if hasattr(model, "model") and getattr(model, "model", None) is not None:
3252
+ model_class = type(model).__name__.lower()
3253
+ if "glm" in model_class:
3254
+ pred = model.predict(data, linear=(pred_type == "link"))
3255
+ else:
3256
+ pred = model.predict(data)
3257
+ else:
3258
+ pred = model.predict(data)
3259
+ except Exception:
3260
+ # scikit-learn style: needs a 2-D array-like input.
3261
+ pred = model.predict(data.values)
3262
+ else:
3263
+ raise ValueError(
3264
+ f"Model of type {type(model).__name__} has no predict method."
3265
+ )
3266
+
3267
+ if isinstance(pred, pd.Series):
3268
+ return pred.reindex(data.index)
3269
+
3270
+ if isinstance(pred, pd.DataFrame):
3271
+ if pred.shape[1] == 1:
3272
+ pred = pred.iloc[:, 0]
3273
+ else:
3274
+ raise ValueError(
3275
+ f"Model predictions must be 1-D, got shape {pred.shape}"
3276
+ )
3277
+ return pred.reindex(data.index)
3278
+
3279
+ pred = np.asarray(pred)
3280
+ if pred.ndim != 1:
3281
+ if pred.shape[1] == 1:
3282
+ pred = pred.ravel()
3283
+ else:
3284
+ raise ValueError(
3285
+ f"Model predictions must be 1-D, got shape {pred.shape}"
3286
+ )
3287
+ return pd.Series(pred, index=data.index)
3288
+
3289
+
3290
+ def added_plot(
3291
+ formula: str,
3292
+ data: pd.DataFrame,
3293
+ lm_formula: Optional[str] = None,
3294
+ method: str = "loess",
3295
+ x: Optional[Union[str, int]] = None,
3296
+ offset: bool = True,
3297
+ **kwargs,
3298
+ ):
3299
+ """Create an added variable plot (R-flexplot ``added.plot()`` parity).
3300
+
3301
+ Residualizes the outcome on a conditioning model, adds the mean of the
3302
+ outcome back to the residuals (R's "maintain interpretation" step), and
3303
+ plots the chosen display variable against those residuals.
3304
+
3305
+ R-flexplot semantics (v0.8.0+; supersedes the v0.6.x behavior which
3306
+ plotted the first variable against doubly-residualized data):
3307
+
3308
+ - ``formula``: ``y ~ var1 + var2 + ...``.
3309
+ - Default display variable (``x=None``) is the **last** variable on the
3310
+ RHS of ``formula`` (R's default): ``y ~ x + z`` residualizes ``y`` on
3311
+ ``x`` and plots ``z``.
3312
+ - ``lm_formula`` (optional): the fitted model used to residualize ``y``
3313
+ (e.g. ``"weight.loss ~ health * muscle.gain"``). Defaults to the
3314
+ remaining formula variables (all but the display variable).
3315
+ - ``x`` (optional): which variable to display. Either the column name
3316
+ or its 1-based position in ``formula``'s predictor list (R uses
3317
+ ``x=2`` for the second).
3318
+ - ``offset`` (default ``True``): add the mean of ``y`` back onto the
3319
+ residuals so the y-axis keeps the outcome's scale (R does this).
3320
+ Pass ``False`` for raw centered residuals.
3321
+ - ``method``: smoother for the fitted line (default ``"loess"``,
3322
+ matching R).
3323
+ """
3324
+ variables = parse_flexplot_formula(formula)
3325
+ _validate_data_for_plot(formula, data, variables, require_numeric_x=True)
3326
+
3327
+ y_var = variables["y"]
3328
+ all_x = [v for v in variables["all_x"] if ":" not in v] # atoms only
3329
+ if not all_x:
3330
+ # Degenerate: only an interaction term (or nothing) — fall back.
3331
+ return flexplot(formula, data, **kwargs)
3332
+
3333
+ # Resolve the display variable (R default: last on the RHS).
3334
+ if x is None:
3335
+ x_var = all_x[-1]
3336
+ elif isinstance(x, int):
3337
+ if not (1 <= x <= len(all_x)):
3338
+ raise ValueError(
3339
+ f"x={x} is out of range; formula has {len(all_x)} predictors."
3340
+ )
3341
+ x_var = all_x[x - 1]
3342
+ elif isinstance(x, str):
3343
+ if x not in all_x:
3344
+ raise ValueError(
3345
+ f"x={x!r} not found among formula predictors {all_x}."
3346
+ )
3347
+ x_var = x
3348
+ else:
3349
+ raise TypeError(f"x must be a str, int, or None; got {type(x).__name__}.")
3350
+
3351
+ # Conditioning variables: from lm_formula if given, else all formula
3352
+ # predictors except the display variable (R: "the fitted model that is
3353
+ # then residualized"). Handle interaction-expanded atoms.
3354
+ if lm_formula is not None:
3355
+ if not isinstance(lm_formula, str):
3356
+ raise TypeError(
3357
+ f"lm_formula must be a string; got {type(lm_formula).__name__}."
3358
+ )
3359
+ lm_variables = parse_flexplot_formula(
3360
+ lm_formula if "~" in lm_formula else f"{y_var} ~ {lm_formula}"
3361
+ )
3362
+ if lm_variables["y"] != y_var:
3363
+ raise ValueError(
3364
+ f"lm_formula {lm_formula!r} must share the outcome {y_var!r}; "
3365
+ f"it uses {lm_variables['y']!r}."
3366
+ )
3367
+ condition_vars = [v for v in lm_variables["all_x"] if ":" not in v]
3368
+ if x_var in condition_vars:
3369
+ condition_vars.remove(x_var)
3370
+ else:
3371
+ condition_vars = [v for v in all_x if v != x_var]
3372
+ if not condition_vars:
3373
+ # Only one predictor: nothing to condition on; plain flexplot.
3374
+ return flexplot(formula, data, **kwargs)
3375
+
3376
+ # Residualize y on the conditioning variables.
3377
+ clean_cond = [re.sub(r"\W", "_", v) for v in condition_vars]
3378
+ mapping = {orig: clean for orig, clean in zip(condition_vars, clean_cond)}
3379
+ needed = list(dict.fromkeys(condition_vars + [x_var, y_var]))
3380
+ fit_df = data[needed].dropna().copy()
3381
+ for orig, clean in mapping.items():
3382
+ if orig != clean:
3383
+ fit_df[clean] = fit_df.pop(orig)
3384
+ y_res_model = OLS.from_formula(
3385
+ f"{y_var} ~ {' + '.join(clean_cond)}", data=fit_df
3386
+ ).fit()
3387
+ y_residuals = y_res_model.resid + y_res_model.model.endog.mean() if offset \
3388
+ else y_res_model.resid
3389
+
3390
+ plot_df = pd.DataFrame({
3391
+ x_var: fit_df[x_var].to_numpy(),
3392
+ f"{y_var}|cond": np.asarray(y_residuals),
3393
+ })
3394
+
3395
+ aes_kwargs = {"x": x_var, "y": f"{y_var}|cond"}
3396
+ p = ggplot(plot_df, aes(**aes_kwargs))
3397
+ p += geom_point(alpha=0.5)
3398
+ # Reuse the numeric-smooth machinery, honoring method / uncertainty /
3399
+ # level / bands kwargs when provided.
3400
+ kwargs_unc = kwargs.pop("uncertainty", "ci")
3401
+ kwargs_level = kwargs.pop("level", 0.95)
3402
+ kwargs_bands = kwargs.pop("bands", None)
3403
+ p = _add_numeric_smooth(
3404
+ p, plot_df, x_var, f"{y_var}|cond",
3405
+ method if method in _VALID_FLEXPLOT_METHODS else ("loess" if method == "auto" else method),
3406
+ kwargs_unc, kwargs_level, kwargs_bands,
3407
+ )
3408
+ p += labs(x=x_var, y=f"{y_var} | conditional", title="Added Variable Plot")
3409
+ p += theme_bw()
3410
+ return p