processbehavior 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. processbehavior/__init__.py +133 -0
  2. processbehavior/analysis.py +2944 -0
  3. processbehavior/analysis_dataset.py +361 -0
  4. processbehavior/analysis_result.py +1276 -0
  5. processbehavior/calibration.py +70 -0
  6. processbehavior/capability.py +720 -0
  7. processbehavior/data_preparation.py +792 -0
  8. processbehavior/datasets/__init__.py +37 -0
  9. processbehavior/datasets/data/coffee_shop_demo_long.csv +385 -0
  10. processbehavior/datasets/loaders.py +66 -0
  11. processbehavior/datasets/synthetic.py +1711 -0
  12. processbehavior/derivations.py +602 -0
  13. processbehavior/effects_calculator.py +603 -0
  14. processbehavior/exceptions.py +216 -0
  15. processbehavior/exporters/__init__.py +5 -0
  16. processbehavior/exporters/excel_exporter.py +708 -0
  17. processbehavior/focused_result.py +329 -0
  18. processbehavior/formulation_spec.py +144 -0
  19. processbehavior/loss_function.py +517 -0
  20. processbehavior/maximum_information.py +285 -0
  21. processbehavior/plotting/__init__.py +46 -0
  22. processbehavior/plotting/_math.py +121 -0
  23. processbehavior/plotting/capability_chart.py +858 -0
  24. processbehavior/plotting/contracts.py +176 -0
  25. processbehavior/plotting/control_chart.py +206 -0
  26. processbehavior/plotting/effects_charts.py +640 -0
  27. processbehavior/plotting/lane_boundaries.py +78 -0
  28. processbehavior/plotting/limits.py +157 -0
  29. processbehavior/plotting/maximum_information_chart.py +395 -0
  30. processbehavior/plotting/plotter.py +1469 -0
  31. processbehavior/plotting/renderers.py +660 -0
  32. processbehavior/plotting/report.py +211 -0
  33. processbehavior/plotting/residuals.py +282 -0
  34. processbehavior/plotting/run_rules_viz.py +150 -0
  35. processbehavior/plotting/stats_box.py +183 -0
  36. processbehavior/plotting/themes.py +544 -0
  37. processbehavior/plotting/x_axis_layout.py +474 -0
  38. processbehavior/plotting/zones.py +127 -0
  39. processbehavior/process_behavior.py +1290 -0
  40. processbehavior/py.typed +0 -0
  41. processbehavior/residual_calculator.py +572 -0
  42. processbehavior/result_signals.py +149 -0
  43. processbehavior/result_tabular.py +296 -0
  44. processbehavior/sds_detector.py +1432 -0
  45. processbehavior/signals/__init__.py +14 -0
  46. processbehavior/signals/config.py +435 -0
  47. processbehavior/signals/detector.py +328 -0
  48. processbehavior/signals/detectors.py +368 -0
  49. processbehavior/signals/result.py +232 -0
  50. processbehavior/spc_constants.py +692 -0
  51. processbehavior/study.py +2733 -0
  52. processbehavior/types.py +154 -0
  53. processbehavior-0.2.0.dist-info/METADATA +279 -0
  54. processbehavior-0.2.0.dist-info/RECORD +56 -0
  55. processbehavior-0.2.0.dist-info/WHEEL +4 -0
  56. processbehavior-0.2.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,2944 @@
1
+ """
2
+ Analysis - Chart calculation strategies for process behavior analysis.
3
+
4
+ This module provides the Analysis class which executes chart calculations using
5
+ the strategy pattern. It supports:
6
+ - Xbar and S charts (subgroup mean and variation)
7
+ - X charts (individual values)
8
+ - mR charts (moving range)
9
+
10
+ The Analysis class coordinates between:
11
+ - FormulationSpec (structural configuration)
12
+ - ChartRequest (per-execute chart parameters)
13
+ - AnalysisDataSet (data preparation and VAS calculations)
14
+ - Chart calculation strategies
15
+ - AnalysisResult (unified result container)
16
+
17
+ Usage:
18
+ spec = FormulationSpec(response_var='Height', rsg_vars=('Operator', 'Machine'), time_var='Time')
19
+ request = ChartRequest(chart='Xbar')
20
+ analysis = Analysis(spec, request, analysis_dataset=ads)
21
+ result = analysis.calculate()
22
+
23
+ # Access charts
24
+ xbar = result.get_chart('Xbar')
25
+ s = result.get_chart('S')
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import logging
31
+ from dataclasses import dataclass
32
+ from functools import reduce
33
+
34
+ import numpy as np
35
+ import pandas as pd
36
+
37
+ from .analysis_dataset import AnalysisDataSet
38
+ from .analysis_result import AnalysisResult
39
+ from .data_preparation import encode_rsg, encode_rsg_series
40
+ from .exceptions import CalibrationNotSupportedError, ChartNotAvailableError, ValidationError
41
+ from .formulation_spec import ChartRequest, FormulationSpec
42
+ from .residual_calculator import calculate_r6_residuals, resolve_r6_groupby
43
+ from .spc_constants import (
44
+ REQUEST_RESIDUALS,
45
+ calculate_limits,
46
+ calculate_limits_vectorized,
47
+ calibrated_limits,
48
+ )
49
+ from .types import ChartPayload
50
+
51
+ # Configure module logger
52
+ logger = logging.getLogger(__name__)
53
+
54
+
55
+ # ============================================================================
56
+ # MR-Family Chart Specification
57
+ # ============================================================================
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class _MRChartSpec:
62
+ """Captures what makes an MR-family chart different from its sibling.
63
+
64
+ X and mR charts share >85% of their calculation pipeline. The differences
65
+ are behavioral, not parametric — this dataclass encodes those differences
66
+ so a single shared method can serve both chart types.
67
+ """
68
+
69
+ chart_type: str # 'X' or 'mR'
70
+ limits_type: str # "XmR" or "R" — passed to calculate_limits()
71
+ plot_col: str # Which column to plot: 'raw' (use value_col) or 'mr'
72
+ center_source: str # What the center line represents: 'mean' or 'mR'
73
+ drops_first_mr: bool # False for X, True for mR
74
+ lane_boundary_offset: int # 0 for X, -1 for mR
75
+
76
+
77
+ def _mr_family_subgroup_n(chart_type: str) -> int:
78
+ """Subgroup size to report in an MR-family chart's statistics dict.
79
+
80
+ An X chart plots one observation per point; an mR chart plots the range of
81
+ a two-observation moving window. Reporting these keeps ``statistics['N']``
82
+ meaning the same thing — subgroup size — on every chart family, which is
83
+ what ``get_statistics`` documents. The X/mR paths used to omit the key
84
+ entirely, so code written against the documented four-key contract raised
85
+ ``KeyError`` on exactly the charts where it was promised.
86
+ """
87
+ return 2 if chart_type == 'mR' else 1
88
+
89
+
90
+ _XMR_SPEC = _MRChartSpec(
91
+ chart_type='X',
92
+ limits_type='XmR',
93
+ plot_col='raw',
94
+ center_source='mean',
95
+ drops_first_mr=False,
96
+ lane_boundary_offset=0,
97
+ )
98
+ _R_SPEC = _MRChartSpec(
99
+ chart_type='mR',
100
+ limits_type='R',
101
+ plot_col='mr',
102
+ center_source='mR',
103
+ drops_first_mr=True,
104
+ lane_boundary_offset=-1,
105
+ )
106
+
107
+
108
+ def _join_limits(grouped: pd.DataFrame, lims_result, rsuffix: str = '') -> pd.DataFrame:
109
+ """Join lpl/upl from a DataFrame.apply(calculate_limits) result.
110
+
111
+ ``DataFrame.apply()`` with ``calculate_limits`` returns either:
112
+ - A ``pd.DataFrame`` (when all Series share the same index), or
113
+ - A ``pd.Series`` of ``pd.Series`` objects.
114
+
115
+ This helper normalizes both cases and joins ``lpl``/``upl`` onto *grouped*.
116
+ """
117
+ if isinstance(lims_result, pd.DataFrame):
118
+ return grouped.join(lims_result[['lpl', 'upl']], rsuffix=rsuffix)
119
+
120
+ # Series of Series — extract lpl/upl manually
121
+ lims_df = pd.DataFrame(
122
+ [(row['lpl'], row['upl']) for row in lims_result],
123
+ index=grouped.index,
124
+ columns=['lpl', 'upl'],
125
+ )
126
+ return grouped.join(lims_df, rsuffix=rsuffix)
127
+
128
+
129
+ # ---------------------------------------------------------------------------
130
+ # MR-chart payload assembly helpers
131
+ # ---------------------------------------------------------------------------
132
+ #
133
+ # The four `_calculate_mr_chart_*` paths (precomputed/independent ×
134
+ # stratified/ungrouped, phased and global) all end with near-identical
135
+ # code that adjusts lane boundaries for the dropped-first-mR row,
136
+ # filters insufficient strata, and assembles the output dict.
137
+ # These helpers consolidate that tail so adding a new metadata key, or
138
+ # fixing a strata-filter bug, happens in one place.
139
+
140
+
141
+ def _shift_lane_boundaries(lane_boundaries, offset: int):
142
+ """Shift each boundary's `position` by `offset`, drop those that go negative.
143
+
144
+ Accepts the flat (`list[dict]`) shape or the stratified
145
+ (`dict[stratum, list[dict]]`) shape. Returns the same shape; returns
146
+ None when the input is empty or every position was dropped.
147
+ """
148
+ if not lane_boundaries:
149
+ return None
150
+ if isinstance(lane_boundaries, dict):
151
+ adjusted: dict[str, list[dict]] = {}
152
+ for stratum, boundaries in lane_boundaries.items():
153
+ shifted = [
154
+ {**b, 'position': b['position'] + offset}
155
+ for b in boundaries
156
+ if b['position'] + offset >= 0
157
+ ]
158
+ if shifted:
159
+ adjusted[stratum] = shifted
160
+ return adjusted or None
161
+ shifted = [
162
+ {**b, 'position': b['position'] + offset}
163
+ for b in lane_boundaries
164
+ if b['position'] + offset >= 0
165
+ ]
166
+ return shifted or None
167
+
168
+
169
+ def _split_strata_by_sufficiency(
170
+ out: pd.DataFrame, stratify_col: str, strata: list, min_obs: int = 2
171
+ ) -> tuple[list, list]:
172
+ """Partition `strata` into (insufficient, published) by post-mR row count.
173
+
174
+ The focus()/strata contract requires every published stratum to be
175
+ focusable; insufficient strata have empty (or near-empty) data after
176
+ the first-mR drop and would fail downstream consumers.
177
+
178
+ Returns (insufficient_strata, published_strata).
179
+
180
+ One counting pass, not one table scan per stratum. The obvious form —
181
+
182
+ [s for s in strata if len(out[out[stratify_col] == s]) < min_obs]
183
+
184
+ compares the whole column and materialises a filtered frame for *every* stratum, which
185
+ is O(strata x rows): 2.4s at 200K rows / 632 strata against 0.006s here, and it was 89%
186
+ of a stratified execute. The `published` membership test uses a set for the same
187
+ reason — against a list it is O(strata^2), invisible at 632 strata and quadratic beyond.
188
+
189
+ NaN strata count as 0 either way: value_counts() drops NaN, and `col == NaN` is always
190
+ False.
191
+ """
192
+ counts = out[stratify_col].value_counts()
193
+ insufficient = [s for s in strata if counts.get(s, 0) < min_obs]
194
+ insufficient_set = set(insufficient)
195
+ published = [s for s in strata if s not in insufficient_set]
196
+ return insufficient, published
197
+
198
+
199
+ def _no_replication_alternative(chart: str) -> str:
200
+ """What to reach for instead, when a subgroup chart has nothing to subgroup."""
201
+ return (
202
+ "Use chart='X' for individual values, or chart='Xbar' with value='R6' for effects analysis."
203
+ if chart == 'Xbar'
204
+ else "Use chart='X' for individual values."
205
+ )
206
+
207
+
208
+ def _raise_no_replicated_subgroups(chart: str, sds) -> None:
209
+ """Raise for an *ungrouped* Xbar/S chart with nothing to subgroup.
210
+
211
+ Sibling of :func:`_raise_no_replication`, which serves the stratified paths and
212
+ adds the per-stratum detail. Both are the enforcement half of the rule
213
+ :meth:`Study._response_pair_problem` reports in advance, so ``why_not()`` and
214
+ ``execute()`` can no longer disagree about whether Xbar can chart the response.
215
+ Change one, change all three.
216
+ """
217
+ raise ValidationError(
218
+ f'No subgroups with n > 1 found — {chart} chart requires replicated observations.\n'
219
+ f'This data has Analytical Design State {sds}.\n'
220
+ f'{_no_replication_alternative(chart)}'
221
+ )
222
+
223
+
224
+ def _raise_no_replication(chart: str, sds, insufficient: list, total: int) -> None:
225
+ """Raise the self-diagnostic error for a subgroup chart with nothing to subgroup.
226
+
227
+ Xbar/S both need n >= 2 within a subgroup; with every subgroup at n = 1 there is no
228
+ within-subgroup dispersion to chart. Shared by the stratified paths so the stratified
229
+ message matches the ungrouped one — the stratified S path used to fall through to
230
+ ``pd.concat([])`` and surface a raw pandas "No objects to concatenate" instead.
231
+ """
232
+ detail = {2: ' (no replication)', 3: ' (partial replication)'}.get(sds, '')
233
+ alternative = _no_replication_alternative(chart)
234
+ raise ValidationError(
235
+ f'No subgroups with n > 1 found — {chart} chart requires replicated observations.\n'
236
+ f'This data has Analytical Design State {sds}{detail}.\n'
237
+ f'All {total} strata were unusable: {insufficient}.\n'
238
+ f'{alternative}'
239
+ )
240
+
241
+
242
+ def _build_stratified_mr_payload(
243
+ *,
244
+ mr_spec: _MRChartSpec,
245
+ chart_out: pd.DataFrame,
246
+ chart_statistics: dict,
247
+ plot_col: str,
248
+ stratify_col: str,
249
+ stratify_by,
250
+ strata: list,
251
+ lane_boundaries,
252
+ phased: bool = False,
253
+ insufficient_strata: list | None = None,
254
+ single_point_phases: int | None = None,
255
+ ) -> dict:
256
+ """Build the {chart_type: {...}} payload for a stratified MR-family chart.
257
+
258
+ For phased layouts `strata` is published as-is. For non-phased
259
+ layouts, callers pre-compute `insufficient_strata` via
260
+ `_split_strata_by_sufficiency` and pass them in; published strata
261
+ are derived here.
262
+ """
263
+ metadata: dict = {
264
+ 'chart_type': mr_spec.chart_type,
265
+ 'value_col': plot_col,
266
+ 'center_col': 'center',
267
+ 'stratified': True,
268
+ 'lane_boundaries': lane_boundaries if lane_boundaries else None,
269
+ 'stratify_col': stratify_col,
270
+ 'stratify_by': list(stratify_by),
271
+ }
272
+ if phased:
273
+ metadata['phased'] = True
274
+ metadata['run_rules_applicable'] = False
275
+ if single_point_phases is not None:
276
+ metadata['single_point_phases'] = single_point_phases
277
+ published_strata = list(strata)
278
+ else:
279
+ metadata['insufficient_strata'] = insufficient_strata or None
280
+ published_strata = [s for s in strata if s not in (insufficient_strata or [])]
281
+ return {
282
+ mr_spec.chart_type: {
283
+ 'data': chart_out,
284
+ 'statistics': chart_statistics,
285
+ 'metadata': metadata,
286
+ 'strata': published_strata,
287
+ },
288
+ }
289
+
290
+
291
+ def _build_ungrouped_mr_payload(
292
+ *,
293
+ mr_spec: _MRChartSpec,
294
+ chart_out: pd.DataFrame,
295
+ chart_statistics: dict,
296
+ plot_col: str,
297
+ lane_boundaries,
298
+ phased: bool = False,
299
+ single_point_phases: int | None = None,
300
+ ) -> dict:
301
+ """Build the {chart_type: {...}} payload for an ungrouped MR-family chart."""
302
+ metadata: dict = {
303
+ 'chart_type': mr_spec.chart_type,
304
+ 'value_col': plot_col,
305
+ 'center_col': 'center',
306
+ 'lane_boundaries': lane_boundaries,
307
+ }
308
+ if phased:
309
+ metadata['phased'] = True
310
+ metadata['run_rules_applicable'] = False
311
+ if single_point_phases is not None:
312
+ metadata['single_point_phases'] = single_point_phases
313
+ return {
314
+ mr_spec.chart_type: {
315
+ 'data': chart_out,
316
+ 'statistics': chart_statistics,
317
+ 'metadata': metadata,
318
+ },
319
+ }
320
+
321
+
322
+ # Chart dispatch tables — added by analysis type and companion mode.
323
+ # Values are method names on Analysis; the orchestrator calls
324
+ # `getattr(self, name)(...)` so this stays a single source of truth for
325
+ # "which method handles this chart type?". Adding a new chart type
326
+ # becomes adding two rows.
327
+ _COMPANION_STRATEGY_MAP: dict[str, str] = {
328
+ 'Xbar': '_calculate_xbar_s',
329
+ 'S': '_calculate_xbar_s',
330
+ 'X': '_calculate_xmr_r',
331
+ 'mR': '_calculate_xmr_r',
332
+ 'Histogram': '_calculate_histogram', # no companion exists; degrade to standalone
333
+ }
334
+
335
+ _SOLO_STRATEGY_MAP: dict[str, str] = {
336
+ 'Xbar': '_calculate_xbar',
337
+ 'S': '_calculate_s',
338
+ 'X': '_calculate_xmr',
339
+ 'mR': '_calculate_r',
340
+ 'Histogram': '_calculate_histogram',
341
+ }
342
+
343
+ # Residual charts call the companion-style methods (Xbar_S / Xmr_R)
344
+ # when companion=True, the solo methods otherwise. Both tables mirror their
345
+ # response-chart counterparts entry for entry — residual context uses the same
346
+ # five base entry points.
347
+ _RESIDUAL_COMPANION_STRATEGY_MAP: dict[str, str] = {
348
+ 'Xbar': '_calculate_xbar_s',
349
+ 'S': '_calculate_xbar_s',
350
+ 'X': '_calculate_xmr_r',
351
+ 'mR': '_calculate_xmr_r',
352
+ 'Histogram': '_calculate_histogram',
353
+ }
354
+
355
+ _RESIDUAL_SOLO_STRATEGY_MAP: dict[str, str] = {
356
+ 'Xbar': '_calculate_xbar',
357
+ 'S': '_calculate_s',
358
+ 'X': '_calculate_xmr',
359
+ # 'mR' was missing here while _SOLO_STRATEGY_MAP had it, so a standalone
360
+ # moving-range chart of residuals was unreachable even though the companion
361
+ # path computed exactly that. See tests/test_solo_mr_residuals.py.
362
+ 'mR': '_calculate_r',
363
+ 'Histogram': '_calculate_histogram',
364
+ }
365
+
366
+
367
+ class Analysis:
368
+ """
369
+ Unified analysis class handling all chart types via strategy pattern.
370
+
371
+ This class replaces the AbstractFactory pattern with a simpler, more maintainable
372
+ approach. All analysis types (Xbar, S, X, mR) are handled through internal
373
+ strategy methods.
374
+
375
+ Usage:
376
+ # Standard usage (calculates AnalysisDataSet from scratch)
377
+ analysis = Analysis(df, specification)
378
+ result = analysis.calculate()
379
+
380
+ # With pre-calculated AnalysisDataSet (for Study.execute())
381
+ analysis = Analysis(df, specification, analysis_dataset=ads)
382
+ result = analysis.calculate() # Reuses pre-calculated data
383
+ """
384
+
385
+ def __init__(
386
+ self,
387
+ spec: FormulationSpec,
388
+ request: ChartRequest,
389
+ analysis_dataset: AnalysisDataSet | None = None,
390
+ sds: int | None = None,
391
+ df: pd.DataFrame | None = None,
392
+ ):
393
+ """
394
+ Initialize analysis with spec and chart request.
395
+
396
+ Args:
397
+ spec: Structural configuration (from formulate())
398
+ request: Chart-specific request (from execute())
399
+ analysis_dataset: Optional pre-calculated AnalysisDataSet.
400
+ If provided, skips expensive residual calculation.
401
+ Used by Study.execute() to reuse formulate() calculations.
402
+ sds: Sampling Design State (0-6). Required if analysis_dataset is not
403
+ provided. SDS should be detected at the entry point (ProcessBehavior)
404
+ and passed through the system.
405
+ df: Raw DataFrame. Required if analysis_dataset is not provided.
406
+
407
+ Raises:
408
+ ValueError: If neither analysis_dataset nor sds is provided.
409
+ """
410
+ self.spec = spec
411
+ self.request = request
412
+ self.analysis_type = request.chart
413
+
414
+ # Calibration (standards-given limits). center_zero applies to a plain
415
+ # residual chart (residual set, not recentered): its mean is meaningless,
416
+ # so the center is pinned at 0. Raw response and recentered residuals use
417
+ # the calibration mean.
418
+ self.calibration = request.calibration
419
+ self._calibration_center_zero = bool(request.residual) and not request.recentered
420
+
421
+ # Per-request frame for request residuals (R6/RCR6): built in calculate(),
422
+ # None for every other request. The shared study-level frame is never
423
+ # mutated — strategies read through _source_frame().
424
+ self._request_frame: pd.DataFrame | None = None
425
+
426
+ # Use pre-calculated AnalysisDataSet if provided, otherwise calculate
427
+ if analysis_dataset is not None:
428
+ self.ads = analysis_dataset
429
+ else:
430
+ if sds is None:
431
+ raise ValidationError(
432
+ 'sds is required when analysis_dataset is not provided. '
433
+ 'SDS should be detected at the entry point (ProcessBehavior) '
434
+ 'and passed to Analysis.'
435
+ )
436
+ if df is None:
437
+ raise ValidationError('df is required when analysis_dataset is not provided.')
438
+ self.ads = AnalysisDataSet(df, spec, observed_sds=sds)
439
+
440
+ def calculate(self) -> AnalysisResult:
441
+ """
442
+ Execute the appropriate analysis strategy and return comprehensive results.
443
+
444
+ Returns
445
+ -------
446
+ AnalysisResult
447
+ Unified result object containing:
448
+ - charts: Chart data and statistics
449
+ - residuals: VAS residuals (R1-R5) if calculated
450
+ - effects: Main effects if calculated
451
+ - interactions: Interaction effects if calculated
452
+ - summary: Comprehensive metadata
453
+ - dataset: Full analysis dataset
454
+
455
+ Raises
456
+ ------
457
+ ValueError
458
+ If analysis_type is not supported
459
+
460
+ Examples
461
+ --------
462
+ >>> result = analysis.calculate()
463
+ >>> xbar = result.get_chart('Xbar')
464
+ >>> if result.has_residuals:
465
+ ... residuals = result.residuals
466
+ """
467
+ # Check if this is a residual chart request
468
+ residual = self.request.residual
469
+ if residual:
470
+ # Residual chart analysis - inline logic (was _calculate_residual_chart)
471
+ chart_type = self.request.residual_chart_type or 'X'
472
+ recentered = self.request.recentered
473
+
474
+ # Request residuals (R6/RCR6) are computed here, onto this request's own
475
+ # frame — the shared study-level frame is never mutated.
476
+ if residual in REQUEST_RESIDUALS:
477
+ self._request_frame = self._build_request_frame(recentered)
478
+
479
+ # Determine column name
480
+ col_prefix = 'RCR' if recentered else 'R'
481
+ residual_num = residual[1:] # Extract number from 'R2', 'R3', 'R10', etc.
482
+ col_name = f'{col_prefix}{residual_num}'
483
+
484
+ # Validate column exists
485
+ if col_name not in self._source_frame().columns:
486
+ available = [
487
+ c for c in self._source_frame().columns if c.startswith('R') or c == self.spec.response_var
488
+ ]
489
+ raise ChartNotAvailableError(
490
+ f"Residual column '{col_name}' not found.\n"
491
+ f'Available columns: {available}\n'
492
+ f"This may indicate the analytical design state (ADS) doesn't support this residual type.",
493
+ chart=col_name,
494
+ available=available,
495
+ )
496
+
497
+ # Validate chart type
498
+ valid_chart_types = {'Xbar', 'S', 'X', 'mR', 'Histogram'}
499
+ if chart_type not in valid_chart_types:
500
+ raise ChartNotAvailableError(
501
+ f"Chart type '{chart_type}' not supported for residual charts.\n"
502
+ f'Valid types: {", ".join(sorted(valid_chart_types))}',
503
+ chart=chart_type,
504
+ available=sorted(valid_chart_types),
505
+ )
506
+
507
+ # Question answered by each residual
508
+ questions = {
509
+ 'R2': 'Is within-subgroup variation stable?',
510
+ 'R3': 'Is there interaction between factors and time?',
511
+ 'R4': 'Does time have a significant effect?',
512
+ 'R5': 'Do factors have a significant effect?',
513
+ }
514
+
515
+ # Dispatch to the appropriate residual-chart calculator via registry.
516
+ companion = self.request.companion
517
+ registry = _RESIDUAL_COMPANION_STRATEGY_MAP if companion else _RESIDUAL_SOLO_STRATEGY_MAP
518
+ method_name = registry.get(chart_type)
519
+ if method_name is None:
520
+ raise ChartNotAvailableError(
521
+ f"Chart type '{chart_type}' not supported for residual charts.\n"
522
+ f'Valid types: {", ".join(sorted(registry))}',
523
+ chart=chart_type,
524
+ available=sorted(registry),
525
+ )
526
+ chart_data = getattr(self, method_name)(value_col=col_name)
527
+ chart_data = self._add_residual_metadata(chart_data, residual, recentered, questions)
528
+
529
+ else:
530
+ # Standard chart analysis — registry dispatch by chart type and companion mode.
531
+ companion = self.request.companion
532
+ registry = _COMPANION_STRATEGY_MAP if companion else _SOLO_STRATEGY_MAP
533
+ method_name = registry.get(self.analysis_type)
534
+ if method_name is None:
535
+ raise ChartNotAvailableError(
536
+ f'Analysis type {self.analysis_type} not supported! '
537
+ f'Valid types: {list(registry.keys())}',
538
+ chart=self.analysis_type,
539
+ available=list(registry.keys()),
540
+ )
541
+ chart_data = getattr(self, method_name)()
542
+
543
+ # Tag calibration provenance for plotting/inspection (no-op when None).
544
+ chart_data = self._apply_calibration_metadata(chart_data)
545
+
546
+ # Wrap in AnalysisResult for unified access
547
+ # Pass analysis_type so result.summary reports the executed chart type, not the recommended one
548
+ return AnalysisResult(
549
+ charts=chart_data,
550
+ analysis_dataset_obj=self.ads,
551
+ analysis_type=self.analysis_type,
552
+ request_dataset=self._request_frame,
553
+ )
554
+
555
+ # =========================================================================
556
+ # Helper Methods (DRY principle)
557
+ # =========================================================================
558
+
559
+ def _source_frame(self) -> pd.DataFrame:
560
+ """The frame this request's strategies read from.
561
+
562
+ The per-request copy carrying R6/RCR6 for a request residual, else the
563
+ shared study-level frame. Callers must ``.copy()`` before mutating,
564
+ exactly as with the shared frame.
565
+ """
566
+ return self._request_frame if self._request_frame is not None else self.ads.analysis_dataset
567
+
568
+ def _build_request_frame(self, recentered: bool) -> pd.DataFrame:
569
+ """Compute this request's R6/RCR6 onto a copy of the study-level frame.
570
+
571
+ R6 = α_i + R2 with α_i = mean(R5 | this request's by= factor(s)) — see
572
+ residual_calculator.calculate_r6_residuals for the math and its
573
+ bit-identity constraints.
574
+ """
575
+ shared = self.ads.analysis_dataset
576
+ missing = [c for c in ('R5', 'R2') if c not in shared.columns]
577
+ if missing:
578
+ available = [c for c in shared.columns if c.startswith('R') or c == self.spec.response_var]
579
+ raise ChartNotAvailableError(
580
+ f'R6 requires the stored residuals {missing} which are not present.\n'
581
+ f'Available columns: {available}\n'
582
+ f"This may indicate the analytical design state (ADS) doesn't support this residual type.",
583
+ chart='R6',
584
+ available=available,
585
+ )
586
+ groupby_key = resolve_r6_groupby(self.request.by, self.spec.rsg_vars_list)
587
+ return calculate_r6_residuals(shared, groupby_key, recentered)
588
+
589
+ def _resolve_xmr_stratification(self) -> tuple[list[str], list[str]]:
590
+ """(stratify_by, collapsed_factors) for the X/mR path.
591
+
592
+ ``by`` means *stratify by* here, not *aggregate by* — the opposite half of
593
+ the dual semantics that :meth:`_resolve_by_grouping` implements for Xbar/S.
594
+
595
+ Single source of truth: both :meth:`_calculate_xmr` and
596
+ :meth:`calibration_rejection_reason` call this. Inlining it in either place
597
+ is how the two answers drift apart.
598
+ """
599
+ rsg_vars = self.spec.rsg_vars_list
600
+ by = list(self.request.by) if self.request.by is not None else None
601
+
602
+ if by is None:
603
+ # No `by` at all: one chart per factor combination.
604
+ return ([self.spec.rsg_var_name] if rsg_vars else []), []
605
+ if by == []:
606
+ # Explicit collapse: one chart, every factor folded into the sequence.
607
+ return [], list(rsg_vars)
608
+ return list(by), [v for v in rsg_vars if v not in by]
609
+
610
+ def calibration_rejection_reason(self) -> str | None:
611
+ """The path description a calibration would be refused for, or None if it applies.
612
+
613
+ Calibration is applied on global and grouped (single-chart) Xbar/S/X/mR paths.
614
+ Stratified paths and phased X/mR are a documented follow-up.
615
+
616
+ This is the predicate behind :meth:`Study.supports_calibration`. It asks the
617
+ same two resolvers the chart code asks — :meth:`_resolve_by_grouping` for
618
+ Xbar/S and :meth:`_resolve_xmr_stratification` for X/mR — rather than
619
+ restating when stratification happens, because that rule is not obvious:
620
+ Xbar/S stratify on ``by=[time]`` **only when charting the response**, so the
621
+ same ``by`` is refused for the response and accepted for a residual.
622
+ """
623
+ residual = self.request.residual
624
+ if residual:
625
+ chart_type = self.request.residual_chart_type or 'X'
626
+ value_col = f'{"RCR" if self.request.recentered else "R"}{residual[1:]}'
627
+ else:
628
+ chart_type = self.analysis_type
629
+ value_col = self.spec.response_var
630
+
631
+ if chart_type in ('Xbar', 'S'):
632
+ _, _, stratify_by = self._resolve_by_grouping(value_col)
633
+ return f'stratified {chart_type}' if stratify_by else None
634
+
635
+ if chart_type in ('X', 'mR'):
636
+ stratify_by, collapsed_factors = self._resolve_xmr_stratification()
637
+ if stratify_by:
638
+ return 'stratified X/mR'
639
+ if self.request.phased and collapsed_factors:
640
+ return 'phased X/mR'
641
+ return None
642
+
643
+ # Histogram and anything else limit-free: nothing for a calibration to change.
644
+ return None
645
+
646
+ def _reject_calibration(self, context: str) -> None:
647
+ """Raise if a calibration is set on a path that does not yet support it.
648
+
649
+ Called at the branch points that would otherwise silently ignore it. The
650
+ same rule, asked ahead of time, is :meth:`calibration_rejection_reason`.
651
+ """
652
+ if self.calibration is not None:
653
+ raise CalibrationNotSupportedError(
654
+ f'calibration is not supported with {context} yet. '
655
+ 'Apply a calibration to a non-stratified, non-phased chart '
656
+ '(e.g. study.execute(chart="X", by=[], calibration=cal)). '
657
+ 'Ask first with study.supports_calibration(chart=..., by=...).',
658
+ context=context,
659
+ )
660
+
661
+ def _calibrated_row(self, limits_type: str, N: int | None = None) -> pd.Series:
662
+ """Calibrated (center, lpl, upl) for one row, as a Series for DataFrame.apply.
663
+
664
+ Delegates to :func:`calibrated_limits`, which reuses ``calculate_limits``
665
+ so rounding/clamping match the data path exactly.
666
+ """
667
+ cal = self.calibration
668
+ center, lims = calibrated_limits(
669
+ limits_type,
670
+ mean=cal.mean,
671
+ sigma=cal.sigma,
672
+ N=N,
673
+ n_sigma=self.request.n_sigma,
674
+ center_zero=self._calibration_center_zero,
675
+ round_to=self.spec.round_to,
676
+ )
677
+ return pd.Series({'center': center, 'lpl': lims['lpl'], 'upl': lims['upl']})
678
+
679
+ def _calibrated_scalar(self, limits_type: str, N: int | None = None) -> tuple[float, pd.Series]:
680
+ """Calibrated (center, limits_series) for a single-stream chart (X/mR).
681
+
682
+ XmR/R take no subgroup size; ``N`` is accepted for symmetry only.
683
+ """
684
+ cal = self.calibration
685
+ return calibrated_limits(
686
+ limits_type,
687
+ mean=cal.mean,
688
+ sigma=cal.sigma,
689
+ N=N,
690
+ n_sigma=self.request.n_sigma,
691
+ center_zero=self._calibration_center_zero,
692
+ round_to=self.spec.round_to,
693
+ )
694
+
695
+ def _apply_calibration_metadata(self, chart_data: dict) -> dict:
696
+ """Tag each chart payload's metadata with the calibration provenance."""
697
+ if self.calibration is None:
698
+ return chart_data
699
+ cal = self.calibration
700
+ badge = {
701
+ 'limits_source': 'calibration',
702
+ 'calibration': {'label': cal.label, 'mean': cal.mean, 'sigma': cal.sigma},
703
+ }
704
+ return {
705
+ key: {**data, 'metadata': {**data.get('metadata', {}), **badge}}
706
+ for key, data in chart_data.items()
707
+ }
708
+
709
+ @staticmethod
710
+ def _add_residual_metadata(chart_data, residual, recentered, questions):
711
+ """Add residual context to chart results. Returns new dict (no mutation)."""
712
+ result = {}
713
+ for chart_key, data in chart_data.items():
714
+ result[chart_key] = {
715
+ **data,
716
+ 'metadata': {
717
+ **data.get('metadata', {}),
718
+ 'residual_type': residual,
719
+ 'recentered': recentered,
720
+ 'question_answered': questions.get(residual, ''),
721
+ },
722
+ }
723
+ return result
724
+
725
+ def _add_beyond_limits_flag(
726
+ self, df: pd.DataFrame, value_col: str, lpl_col: str = 'lpl', upl_col: str = 'upl'
727
+ ) -> pd.DataFrame:
728
+ """
729
+ Add beyond_limits flag column.
730
+
731
+ Returns -1 for below LPL, 1 for above UPL, 0 for in control.
732
+
733
+ Pure function: doesn't modify input.
734
+
735
+ Parameters
736
+ ----------
737
+ df : pd.DataFrame
738
+ Input data with control limits
739
+ value_col : str
740
+ Column name containing values to check
741
+ lpl_col : str, default 'lpl'
742
+ Column name for lower process limit
743
+ upl_col : str, default 'upl'
744
+ Column name for upper process limit
745
+
746
+ Returns
747
+ -------
748
+ pd.DataFrame
749
+ Data with added 'beyond_limits' column
750
+ """
751
+ result = df.copy()
752
+
753
+ # Vectorized form of detect_beyond_limits, which stays the scalar reference (the
754
+ # Bishop validator exercises it, so it should not become a check of this against
755
+ # itself). One Python call per *observation* was the dominant cost of execute() on
756
+ # X/mR charts — 2.14s at 1M rows, versus 0.002s here.
757
+ #
758
+ # The nesting order matters: the scalar rule tests `x < lpl` FIRST, then `x > upl`.
759
+ # NaN compares False both ways in numpy exactly as in Python, so a NaN observation
760
+ # or a NaN limit still yields 0.
761
+ values = result[value_col].to_numpy()
762
+ lower = result[lpl_col].to_numpy()
763
+ upper = result[upl_col].to_numpy()
764
+ result['beyond_limits'] = np.where(values < lower, -1, np.where(values > upper, 1, 0))
765
+
766
+ return result
767
+
768
+ def _determine_n_to_use(self, df: pd.DataFrame, n_col: str = 'n') -> tuple[str, int]:
769
+ """
770
+ Determine if subgroup sizes are constant or variable.
771
+
772
+ Parameters
773
+ ----------
774
+ df : pd.DataFrame
775
+ Input data with subgroup size column
776
+ n_col : str, default 'n'
777
+ Column name containing subgroup sizes
778
+
779
+ Returns
780
+ -------
781
+ tuple[str, int]
782
+ (n_to_use, max_n) where:
783
+ - n_to_use: 'N' if constant sizes, 'n' if variable
784
+ - max_n: maximum subgroup size
785
+
786
+ Examples
787
+ --------
788
+ >>> n_to_use, max_n = self._determine_n_to_use(out)
789
+ >>> # Use max_n for statistics, n_to_use for per-row calculations
790
+ """
791
+ n_max = df[n_col].max()
792
+ n_to_use = 'N' if df[n_col].eq(n_max).all() else 'n'
793
+
794
+ logger.debug('Analysis using %s for calculations (Scenario: %s)', n_to_use, 1 if n_to_use == 'N' else 2)
795
+
796
+ return n_to_use, n_max
797
+
798
+ def _residual_grain(self, value_col: str) -> list[str]:
799
+ """Return the cell-grid columns at which to compute Bishop's grand mean
800
+ for an Xbar chart on this column.
801
+
802
+ Each VAS residual is recentered around a baseline computed at a specific
803
+ grain (see `analysis_dataset.py:364-371` and `residual_calculator.calculate_r6_residuals`). The
804
+ canonical grand mean of an RCRk chart averages over that grain — equal
805
+ weight per cell — yielding Bishop's unweighted center line.
806
+
807
+ Grain table (None for non-residuals; falls through to other logic):
808
+ R1, R2, R3 -> [rsg_var_name, time_var] (full cell grid)
809
+ R4 -> [rsg_var_name] (time effect removed)
810
+ R5, R6 -> [rsg_var_name] (factor effects live at rsg level)
811
+
812
+ Returns an empty list when the column isn't a residual or the spec
813
+ lacks the required factors/time — caller falls back to `df.mean()`.
814
+ """
815
+ if value_col is None:
816
+ return []
817
+ spec = self.spec
818
+ base = value_col.upper()
819
+ if base.startswith('RCR'):
820
+ base = 'R' + base[3:]
821
+ if not (base.startswith('R') and len(base) >= 2 and base[1].isdigit()):
822
+ return []
823
+ # R5/R6 are factor-effect residuals; R4 is the time-effect residual
824
+ # (time component removed). For our SDS 3 validation only R3 and R6 are
825
+ # exercised; R4/R5 grains are inferred from the recentering structure.
826
+ if base in ('R5', 'R6'):
827
+ return [spec.rsg_var_name] if spec.rsg_var_name else []
828
+ if base == 'R4':
829
+ return [spec.rsg_var_name] if spec.rsg_var_name else []
830
+ # R1, R2, R3: full (rsg x time) cell grid
831
+ grain = []
832
+ if spec.rsg_var_name:
833
+ grain.append(spec.rsg_var_name)
834
+ if spec.time_var:
835
+ grain.append(spec.time_var)
836
+ return grain
837
+
838
+ @staticmethod
839
+ def _resolve_limits_column(value_col: str, df: pd.DataFrame) -> str:
840
+ """Return the column to use for within-group std in limit calculations.
841
+
842
+ For effect-carrying residuals (R1/R3/R4/R5 and RCR variants), use R2 instead.
843
+ Non-R2 residuals retain structural effects (factor, time, interaction)
844
+ whose within-group std inflates limits. R2 (within-cell noise) is the
845
+ correct dispersion basis per Bishop.
846
+
847
+ Always uses plain R2 (not RCR2) because recentered residuals add
848
+ cell-specific offsets that inflate within-group std when groups span
849
+ multiple cells.
850
+ """
851
+ _EFFECT_RESIDUALS = {'R1', 'R3', 'R4', 'R5', 'RCR1', 'RCR3', 'RCR4', 'RCR5'}
852
+ if value_col is not None and value_col.upper() in _EFFECT_RESIDUALS and 'R2' in df.columns:
853
+ return 'R2'
854
+ return value_col
855
+
856
+ def _resolve_by_grouping(self, value_col: str) -> tuple[list[str], str | None, list[str]]:
857
+ """
858
+ Resolve groupby columns and pre-calculated Ybar column based on `by` parameter.
859
+
860
+ The `by` parameter controls aggregation level for Xbar/S charts.
861
+ When possible, we use pre-calculated Ybar columns from the analytic dataset.
862
+
863
+ Parameters
864
+ ----------
865
+ value_col : str
866
+ Column being charted (response_var or residual column)
867
+
868
+ Returns
869
+ -------
870
+ tuple[list[str], str | None, list[str]]
871
+ (groupby_cols, ybar_col, stratify_by) where:
872
+ - groupby_cols: columns to group by (empty list for collapse all)
873
+ - ybar_col: pre-calculated Ybar column to use, or None if must aggregate
874
+ - stratify_by: columns to stratify by (separate charts per combo),
875
+ empty list if no stratification
876
+
877
+ Examples
878
+ --------
879
+ >>> groupby_cols, ybar_col, stratify_by = self._resolve_by_grouping('y')
880
+ >>> # by=[] (normalized to by=None) -> ([rsg, time_var], 'Ybar_kt', [])
881
+ >>> # - cell-level aggregation when factors+time exist
882
+ >>> # by=['factor1','factor2'] -> (['rsg'], 'Ybar_k', []) - use factor means
883
+ >>> # by=['time_var'] with factors -> ([time_var], 'Ybar_t', [rsg_var_name])
884
+ >>> # NOTE: ybar_col is None for residual value_col (no pre-cached mean).
885
+ """
886
+ spec = self.spec
887
+ by = self.request.by
888
+ # Normalize tuple to list for internal comparison
889
+ by = list(by) if by is not None else None
890
+ rsg_vars = spec.rsg_vars_list
891
+ time_var = spec.time_var
892
+
893
+ # Determine if we're charting response (can use pre-calculated) or residual
894
+ is_response = value_col == spec.response_var
895
+
896
+ # Xbar/S: by=[] normalizes to by=None (Kt-level aggregation).
897
+ # X/mR handles by=[] directly in _calculate_xmr() as single-stream.
898
+ # This split is intentional — `by` means "aggregate by" for Xbar/S
899
+ # and "stratify by" for X/mR, matching each chart type's semantics.
900
+ if by == []:
901
+ by = None
902
+
903
+ # Default: by=None means use cell_key level (factors + time)
904
+ if by is None:
905
+ if time_var and rsg_vars:
906
+ # Factors + time (Kt level) - matches cell_key
907
+ groupby_cols = rsg_vars + [time_var]
908
+ ybar_col = 'Ybar_kt' if is_response else None
909
+ elif rsg_vars:
910
+ # Factors only (no time)
911
+ groupby_cols = [spec.rsg_var_name]
912
+ ybar_col = 'Ybar_k' if is_response else None
913
+ elif time_var:
914
+ # Time only (no factors)
915
+ groupby_cols = [time_var]
916
+ ybar_col = 'Ybar_t' if is_response else None
917
+ else:
918
+ # No grouping
919
+ groupby_cols = []
920
+ ybar_col = None
921
+ return groupby_cols, ybar_col, []
922
+
923
+ # Check if by matches known aggregation levels for Ybar optimization
924
+ by_set = set(by)
925
+ rsg_set = set(rsg_vars)
926
+
927
+ # by == all factors (rsg_key level) -> use Ybar_k optimization
928
+ # Only if order matches rsg_vars; otherwise preserve user's order
929
+ if by_set == rsg_set and list(by) == rsg_vars:
930
+ groupby_cols = [spec.rsg_var_name]
931
+ ybar_col = 'Ybar_k' if is_response else None
932
+ return groupby_cols, ybar_col, []
933
+
934
+ # by == [time_var] only -> use Ybar_t
935
+ # When factors exist and charting response, also stratify by factor combos
936
+ if by_set == {time_var}:
937
+ groupby_cols = [time_var]
938
+ ybar_col = 'Ybar_t' if is_response else None
939
+ stratify_by = [spec.rsg_var_name] if (rsg_vars and is_response) else []
940
+ return groupby_cols, ybar_col, stratify_by
941
+
942
+ # by == all factors + time (cell_key level) -> use Ybar_kt
943
+ cell_key_vars = rsg_set | ({time_var} if time_var else set())
944
+ if by_set == cell_key_vars:
945
+ # Group by all factors + time
946
+ groupby_cols = rsg_vars + [time_var] if time_var else rsg_vars
947
+ ybar_col = 'Ybar_kt' if is_response else None
948
+ return groupby_cols, ybar_col, []
949
+
950
+ # Partial subset - must aggregate at runtime
951
+ # Use the by columns directly
952
+ groupby_cols = list(by)
953
+ return groupby_cols, None, []
954
+
955
+ def _calculate_lane_boundaries(self, df: pd.DataFrame, collapsed_vars: list[str]) -> list[dict]:
956
+ """
957
+ Calculate lane boundaries where collapsed factors change.
958
+
959
+ Lane boundaries are positions in the data where a factor that was
960
+ "collapsed" (not in `by`) changes value. These are rendered as
961
+ vertical dashed lines on X charts.
962
+
963
+ Parameters
964
+ ----------
965
+ df : pd.DataFrame
966
+ Data in display order (already sorted)
967
+ collapsed_vars : list[str]
968
+ Variables that were collapsed (not in `by`)
969
+
970
+ Returns
971
+ -------
972
+ list[dict]
973
+ List of boundary dicts with 'position' (x index) and 'label' (factor value)
974
+
975
+ Examples
976
+ --------
977
+ >>> # If by=['factor1'] and data has factor1='A' with factor2 changing X→Y
978
+ >>> boundaries = self._calculate_lane_boundaries(df, ['factor2'])
979
+ >>> # Returns: [{'position': 5, 'label': 'Y', 'variable': 'factor2'}]
980
+ """
981
+ if not collapsed_vars or df.empty:
982
+ return []
983
+
984
+ # Combine column-wise rather than with `.agg('_'.join, axis=1)`, which ran once per
985
+ # row (1.05s at 1M). Deliberately NOT encode_rsg_series: that renders a datetime as
986
+ # str(Timestamp) ('2024-01-01 00:00:00') where DataFrame.astype(str) gives
987
+ # '2024-01-01'. These are display labels for a chart divider, not stratum identity,
988
+ # so they keep the astype rendering — do not "unify" the two.
989
+ parts = [df[col].astype(str) for col in collapsed_vars]
990
+ combined = parts[0] if len(parts) == 1 else reduce(lambda left, right: left + '_' + right, parts)
991
+
992
+ # Positional comparison against the previous row. The old form built a boolean mask,
993
+ # took df.index[mask], dropped the first entry, then called df.index.get_loc() per
994
+ # boundary — and get_loc is not O(1) on these frames (~1.3ms a call, ~1.8s total).
995
+ # Comparing the array to itself shifted gives the 0-based positions directly.
996
+ labels = combined.to_numpy()
997
+ changed = np.empty(len(labels), dtype=bool)
998
+ changed[0] = False # the first row is a start, not a boundary
999
+ changed[1:] = labels[1:] != labels[:-1]
1000
+
1001
+ return [
1002
+ {
1003
+ 'position': int(pos), # 0-based position in the sorted data
1004
+ 'label': labels[pos],
1005
+ 'variables': collapsed_vars,
1006
+ }
1007
+ for pos in np.flatnonzero(changed)
1008
+ ]
1009
+
1010
+ def _build_output_columns(self, df: pd.DataFrame, value_cols: list[str]) -> pd.DataFrame:
1011
+ """
1012
+ Select output columns with appropriate time/grouping columns.
1013
+
1014
+ Handles the pattern:
1015
+ - If has_time and has_grouping: [time, rsg, ...values, obs_id]
1016
+ - If has_time only: [time, ...values, obs_id]
1017
+ - If has_grouping only: [x, rsg, ...values, obs_id] (with generated x)
1018
+ - Otherwise: [x, ...values, obs_id] (with generated x)
1019
+
1020
+ Always includes obs_id (if available) for traceability of violations.
1021
+
1022
+ Parameters
1023
+ ----------
1024
+ df : pd.DataFrame
1025
+ Input data
1026
+ value_cols : list[str]
1027
+ Columns to keep (e.g., ['mean', 'lpl', 'upl', 'beyond_limits'])
1028
+
1029
+ Returns
1030
+ -------
1031
+ pd.DataFrame
1032
+ Data with selected columns, rounded
1033
+
1034
+ Notes
1035
+ -----
1036
+ The obs_id column enables tracing violations back to original data:
1037
+ - Find violations: df[df['beyond_limits'] != 0]
1038
+ - Get obs_ids: violation_ids = df['obs_id'].tolist()
1039
+ - Trace to raw data: raw_df[raw_df['obs_id'].isin(violation_ids)]
1040
+
1041
+ Examples
1042
+ --------
1043
+ >>> out = self._build_output_columns(
1044
+ ... df=out,
1045
+ ... value_cols=[spec.response_var, 'mean', 'lpl', 'upl', 'beyond_limits']
1046
+ ... )
1047
+ """
1048
+ result = df.copy()
1049
+ cols_to_keep = value_cols.copy()
1050
+
1051
+ if self.spec.has_time:
1052
+ if self.spec.has_grouping:
1053
+ cols_to_keep.insert(0, self.spec.rsg_var_name)
1054
+ cols_to_keep.insert(0, self.spec.time_var)
1055
+ else:
1056
+ cols_to_keep.insert(0, self.spec.time_var)
1057
+ else:
1058
+ # No explicit time variable specified - use implicit ordering
1059
+ if self.spec.has_grouping:
1060
+ # Per-group sequence number for grouped data
1061
+ result['x'] = result.groupby(self.spec.rsg_var_name, observed=True).cumcount() + 1
1062
+ cols_to_keep.insert(0, self.spec.rsg_var_name)
1063
+ cols_to_keep.insert(0, 'x')
1064
+ else:
1065
+ # Use obs_id as implicit time for single condition over time (SDS 4)
1066
+ # Rationale: Wheeler's X chart assumes temporal ordering, and obs_id
1067
+ # provides that ordering from the original data sequence.
1068
+ # See: ProcessBehavior.formulate() docstring for full explanation.
1069
+ cols_to_keep.insert(0, 'obs_id')
1070
+
1071
+ # Always include obs_id for traceability (if available)
1072
+ # This enables linking violations back to original data
1073
+ if 'obs_id' in result.columns and 'obs_id' not in cols_to_keep:
1074
+ cols_to_keep.append('obs_id')
1075
+
1076
+ return result[cols_to_keep].round(self.spec.round_to)
1077
+
1078
+ # =========================================================================
1079
+ # Chart Calculation Methods (Strategy Pattern)
1080
+ # =========================================================================
1081
+
1082
+ def _calculate_xbar( # noqa: C901
1083
+ self, value_col: str | None = None, _return_intermediates: bool = False
1084
+ ) -> dict:
1085
+ """
1086
+ Calculate Xbar (mean) chart statistics.
1087
+
1088
+ Parameters
1089
+ ----------
1090
+ value_col : str, optional
1091
+ Column to use for calculations. Defaults to spec.response_var.
1092
+ Pass a residual column (R2, R5, etc.) for residual charts.
1093
+ _return_intermediates : bool, optional
1094
+ If True, includes '_intermediates' key with values needed by
1095
+ _calculate_s() to avoid redundant computation. Used internally
1096
+ by _calculate_xbar_s() for companion chart calculation.
1097
+
1098
+ Returns
1099
+ -------
1100
+ dict
1101
+ Chart data: {'Xbar': {'data': df, 'statistics': dict, 'metadata': dict}}
1102
+ If _return_intermediates=True, also includes '_intermediates' key.
1103
+
1104
+ Logic moved from Xbar.calculate_statistics()
1105
+ """
1106
+ spec = self.spec
1107
+ if value_col is None:
1108
+ value_col = spec.response_var
1109
+
1110
+ result = {}
1111
+ statistics = {}
1112
+ df = self._source_frame().copy()
1113
+
1114
+ logger.debug('In calculate statistics XbarS')
1115
+ logger.debug('Dataframe has columns: %s', df.columns.to_list())
1116
+ logger.debug('Dataframe head:\n%s', df.head(10))
1117
+
1118
+ # Resolve grouping based on `by` parameter
1119
+ groupby_cols, ybar_col, stratify_by = self._resolve_by_grouping(value_col)
1120
+
1121
+ # Stratified path: separate chart per factor combination
1122
+ if stratify_by:
1123
+ self._reject_calibration('stratified Xbar')
1124
+ return self._calculate_xbar_stratified(
1125
+ value_col,
1126
+ groupby_cols,
1127
+ stratify_by,
1128
+ _return_intermediates=_return_intermediates,
1129
+ )
1130
+
1131
+ _limits_col = self._resolve_limits_column(value_col, df)
1132
+
1133
+ # Bishop VAS grand mean: equal weight per cell at the residual's natural
1134
+ # grain, regardless of cell N. The unweighted form is canonical; weighted
1135
+ # vs unweighted differ only on unbalanced designs, and Bishop's principle
1136
+ # is that the practical difference is negligible -- the methodology still
1137
+ # requires the unweighted form. Each residual is recentered at a specific
1138
+ # grain (analysis_dataset.py:364-371; residual_calculator.calculate_r6_residuals for R6), and the grand
1139
+ # mean averages over that grain:
1140
+ # R3 (interaction) -> (rsg, time) full cell grid
1141
+ # R4 (time effect) -> (rsg) factor grain (time component removed)
1142
+ # R5, R6 (factor effects) -> (time) or (rsg) depending on what's removed
1143
+ # In practice the only residuals exercised by Bishop's validation are R3
1144
+ # (full cell grid) and R5/R6 (factor grain).
1145
+ if value_col == spec.response_var and 'Ybar' in df.columns:
1146
+ _Ybar = df['Ybar'].iloc[0]
1147
+ else:
1148
+ grain_cols = self._residual_grain(value_col)
1149
+ if grain_cols and value_col in df.columns:
1150
+ _Ybar = df.groupby(grain_cols, observed=True)[value_col].mean().mean()
1151
+ else:
1152
+ _Ybar = df[value_col].mean()
1153
+
1154
+ # Handle by=[] (collapse all) - single point chart
1155
+ if groupby_cols == []:
1156
+ # Single aggregation across all data
1157
+ _S = df[_limits_col].std()
1158
+ _N = len(df)
1159
+ out_dict = {'group': ['All'], 'xbar': [_Ybar], 's': [_S], 'n': [_N], 'N': [_N]}
1160
+ if _limits_col != value_col:
1161
+ out_dict['s_value'] = [df[value_col].std()]
1162
+ out = pd.DataFrame(out_dict)
1163
+ else:
1164
+ # Group by specified columns
1165
+ agg_dict = {
1166
+ 's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
1167
+ 'mean': pd.NamedAgg(column=value_col, aggfunc='mean'),
1168
+ # Count on response_var (not value_col) to avoid NaN issues with residuals
1169
+ 'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
1170
+ }
1171
+ if _limits_col != value_col:
1172
+ agg_dict['s_value'] = pd.NamedAgg(column=value_col, aggfunc='std')
1173
+ out = df.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
1174
+
1175
+ # If we have pre-calculated Ybar, use it for xbar values
1176
+ if ybar_col and ybar_col in df.columns:
1177
+ # Get unique Ybar values per group
1178
+ ybar_by_group = df.groupby(groupby_cols, observed=True)[ybar_col].first().reset_index()
1179
+ out = out.merge(ybar_by_group, on=groupby_cols, how='left')
1180
+ out['xbar'] = out[ybar_col]
1181
+ out = out.drop(columns=[ybar_col, 'mean'], errors='ignore')
1182
+ else:
1183
+ # Rename columns for consistency: mean→xbar (values)
1184
+ out = out.rename(columns={'mean': 'xbar'})
1185
+
1186
+ _N = out['n'].max()
1187
+ out['N'] = _N
1188
+
1189
+ # Filter out groups with n=1 (can't compute c4 for variance estimation)
1190
+ # Xbar charts require n >= 2 for within-group variance.
1191
+ # Xbar stays in valid_charts on ADS 2 — the chart family is valid there
1192
+ # (Xbar of a residual, or of a pooled subgroup, computes fine); it is
1193
+ # only charting *the response* that has nothing to subgroup. See
1194
+ # Study._response_pair_problem, which reports that in advance.
1195
+ mask_n1 = out['n'].eq(1)
1196
+ if mask_n1.any():
1197
+ n_filtered = mask_n1.sum()
1198
+ logger.info(f'Filtered {n_filtered} subgroup(s) with n=1 from Xbar calculation')
1199
+ out = out[~mask_n1].copy()
1200
+
1201
+ # Handle case where no subgroups have >1 observation
1202
+ if out.shape[0] == 0:
1203
+ sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
1204
+ _raise_no_replicated_subgroups('Xbar', sds)
1205
+
1206
+ # Use Bishop VAS grand mean (mean of cell means on value_col) as center
1207
+ _Xbar = _Ybar
1208
+ _S = out['s'].mean()
1209
+ _N = out['n'].max()
1210
+ if 'N' not in out.columns:
1211
+ out['N'] = _N
1212
+
1213
+ # Determine if subgroup sizes are constant or variable
1214
+ n_to_use, n_max = self._determine_n_to_use(out)
1215
+
1216
+ # Override n_to_use if n_mode="average"
1217
+ n_mode = self.request.n_mode
1218
+ n_avg = None
1219
+ if n_mode == 'average':
1220
+ n_avg = out['n'].mean()
1221
+ out['N'] = n_avg # overwrite N column with average
1222
+ n_to_use = 'N' # force constant-N path
1223
+
1224
+ # CALCULATE XBAR
1225
+ xbar = out.copy()
1226
+ if self.calibration is not None:
1227
+ # Standards-given Xbar: center=mean, mean ± n_sigma·sigma/√N (per-row N).
1228
+ xbar[['center', 'lpl', 'upl']] = xbar.apply(
1229
+ lambda row: self._calibrated_row('Xbar', N=row[n_to_use]),
1230
+ axis=1,
1231
+ )
1232
+ else:
1233
+ xbar['center'] = _Xbar # Add center column for Xbar chart
1234
+ xbar[['lpl', 'upl']] = calculate_limits_vectorized(
1235
+ 'Xbar',
1236
+ mean=xbar['center'],
1237
+ sd=_S,
1238
+ N=xbar[n_to_use],
1239
+ sigma_multiplier=self.request.n_sigma,
1240
+ )
1241
+
1242
+ # Detect beyond limits signals
1243
+ xbar = self._add_beyond_limits_flag(xbar, value_col='xbar')
1244
+ xbar = xbar.round(spec.round_to)
1245
+
1246
+ _center_stat = xbar['center'].iloc[0] if self.calibration is not None else _Xbar
1247
+ statistics['center'] = round(_center_stat, spec.round_to)
1248
+ if n_to_use == 'N':
1249
+ statistics['N'] = round(n_avg, spec.round_to) if n_avg is not None else _N
1250
+ statistics['upl'] = xbar['upl'].max()
1251
+ statistics['lpl'] = xbar['lpl'].max()
1252
+ else:
1253
+ statistics['N'] = None
1254
+ statistics['lpl'] = None
1255
+ statistics['upl'] = None
1256
+ statistics['limits_vary'] = True
1257
+
1258
+ # Determine the grouping column for output
1259
+ by = self.request.by
1260
+ if by is not None and len(by) > 0:
1261
+ # Explicit by list
1262
+ if len(by) == 1:
1263
+ # Single factor: preserve original column name for axis labeling.
1264
+ # Cast to string so Plotly treats coded factors (e.g., 1,2,3) as categories.
1265
+ xbar[groupby_cols[0]] = xbar[groupby_cols[0]].astype(str)
1266
+ group_col = groupby_cols[0]
1267
+ else:
1268
+ xbar['subgroup'] = xbar[groupby_cols].astype(str).agg('_'.join, axis=1)
1269
+ group_col = 'subgroup'
1270
+ elif len(groupby_cols) > 1:
1271
+ # by=None - create 'group' from multiple columns (original behavior)
1272
+ xbar['group'] = xbar[groupby_cols].astype(str).agg('_'.join, axis=1)
1273
+ group_col = 'group'
1274
+ elif groupby_cols:
1275
+ # Single groupby column
1276
+ group_col = groupby_cols[0]
1277
+ else:
1278
+ group_col = None
1279
+
1280
+ cols_to_keep = [group_col, 'xbar', 'center', 'lpl', 'upl', 'beyond_limits']
1281
+ cols_to_keep = [c for c in cols_to_keep if c in xbar.columns]
1282
+ xbar = xbar[cols_to_keep]
1283
+ result['Xbar'] = {
1284
+ 'data': xbar,
1285
+ 'statistics': statistics,
1286
+ 'metadata': {
1287
+ 'chart_type': 'Xbar',
1288
+ 'value_col': 'xbar',
1289
+ 'center_col': 'center',
1290
+ 'n_sigma': self.request.n_sigma,
1291
+ 'n_mode': n_mode,
1292
+ **({'n_avg': round(n_avg, spec.round_to)} if n_avg is not None else {}),
1293
+ },
1294
+ }
1295
+
1296
+ # Optionally include intermediates for companion calculation
1297
+ if _return_intermediates:
1298
+ result['_intermediates'] = {
1299
+ '_S': _S,
1300
+ 'n_to_use': n_to_use,
1301
+ 'n_max': n_max,
1302
+ 'groupby_cols': groupby_cols,
1303
+ 'group_col': group_col,
1304
+ 'out': out, # Pre-aggregated DataFrame with s, n, mean columns
1305
+ 'n_avg': n_avg,
1306
+ }
1307
+
1308
+ return result
1309
+
1310
+ def _calculate_xbar_stratified(
1311
+ self,
1312
+ value_col: str,
1313
+ groupby_cols: list[str],
1314
+ stratify_by: list[str],
1315
+ _return_intermediates: bool = False,
1316
+ ) -> dict:
1317
+ """
1318
+ Stratified Xbar chart: one chart per factor combination, time on x-axis.
1319
+
1320
+ Each stratum computes its own Sbar and limits independently.
1321
+
1322
+ Parameters
1323
+ ----------
1324
+ value_col : str
1325
+ Column to chart (response variable).
1326
+ groupby_cols : list[str]
1327
+ Columns to group by within each stratum (typically [time_var]).
1328
+ stratify_by : list[str]
1329
+ Columns defining strata (typically [rsg_var_name]).
1330
+ _return_intermediates : bool
1331
+ If True, include intermediates for companion S chart.
1332
+
1333
+ Returns
1334
+ -------
1335
+ dict
1336
+ Chart data with 'strata' key for stratified output.
1337
+ """
1338
+ spec = self.spec
1339
+ df = self._source_frame().copy()
1340
+ result = {}
1341
+
1342
+ # Determine stratification column
1343
+ if len(stratify_by) == 1:
1344
+ stratify_col = stratify_by[0]
1345
+ else:
1346
+ df['_stratify_key'] = encode_rsg_series(df, stratify_by)
1347
+ stratify_col = '_stratify_key'
1348
+
1349
+ _limits_col = self._resolve_limits_column(value_col, df)
1350
+
1351
+ strata = df[stratify_col].unique().tolist()
1352
+ all_xbar_frames = []
1353
+ chart_statistics = {}
1354
+ intermediates_per_stratum = {}
1355
+ # Strata with no replicated subgroup. Recorded rather than silently skipped: a
1356
+ # stratum published in `strata` but absent from `statistics`/`data` fails
1357
+ # result.focus(), which is the contract _split_strata_by_sufficiency exists to
1358
+ # keep on the X/mR path. Xbar/S never adopted it.
1359
+ insufficient_strata: list = []
1360
+
1361
+ for stratum in strata:
1362
+ sdf = df[df[stratify_col] == stratum].copy()
1363
+
1364
+ # Group by time within this stratum
1365
+ # Note: we always compute mean directly from filtered data, NOT using
1366
+ # Ybar_t (which is the marginal time mean across ALL factors).
1367
+ agg_dict = {
1368
+ 's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
1369
+ 'xbar': pd.NamedAgg(column=value_col, aggfunc='mean'),
1370
+ 'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
1371
+ }
1372
+ if _limits_col != value_col:
1373
+ agg_dict['s_value'] = pd.NamedAgg(column=value_col, aggfunc='std')
1374
+ out = sdf.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
1375
+
1376
+ out['N'] = out['n'].max()
1377
+
1378
+ # Filter subgroups with n=1
1379
+ mask_n1 = out['n'].eq(1)
1380
+ if mask_n1.any():
1381
+ out = out[~mask_n1].copy()
1382
+
1383
+ if out.shape[0] == 0:
1384
+ insufficient_strata.append(stratum)
1385
+ continue
1386
+
1387
+ # Per-stratum statistics
1388
+ _Xbar = out['xbar'].mean()
1389
+ _S = out['s'].mean()
1390
+ n_to_use, n_max = self._determine_n_to_use(out)
1391
+
1392
+ n_mode = self.request.n_mode
1393
+ n_avg = None
1394
+ if n_mode == 'average':
1395
+ n_avg = out['n'].mean()
1396
+ out['N'] = n_avg
1397
+ n_to_use = 'N'
1398
+
1399
+ out['center'] = _Xbar
1400
+ out[['lpl', 'upl']] = calculate_limits_vectorized(
1401
+ 'Xbar',
1402
+ mean=out['center'],
1403
+ sd=_S,
1404
+ N=out[n_to_use],
1405
+ sigma_multiplier=self.request.n_sigma,
1406
+ )
1407
+
1408
+ out = self._add_beyond_limits_flag(out, value_col='xbar')
1409
+ out = out.round(spec.round_to)
1410
+
1411
+ # Add stratum identifier
1412
+ out[stratify_col] = stratum
1413
+
1414
+ # Cast time column to string for categorical x-axis
1415
+ if groupby_cols:
1416
+ out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
1417
+
1418
+ all_xbar_frames.append(out)
1419
+
1420
+ _has_fixed_n = n_to_use == 'N'
1421
+ chart_statistics[stratum] = {
1422
+ 'center': round(_Xbar, spec.round_to),
1423
+ 'N': round(n_avg, spec.round_to) if n_avg is not None else (out['N'].iloc[0] if _has_fixed_n else None),
1424
+ 'lpl': round(out['lpl'].iloc[0], spec.round_to) if _has_fixed_n else None,
1425
+ 'upl': round(out['upl'].iloc[0], spec.round_to) if _has_fixed_n else None,
1426
+ **({'limits_vary': True} if not _has_fixed_n else {}),
1427
+ }
1428
+
1429
+ if _return_intermediates:
1430
+ intermediates_per_stratum[stratum] = {
1431
+ '_S': _S,
1432
+ 'n_to_use': n_to_use,
1433
+ 'n_max': n_max,
1434
+ 'out': out,
1435
+ 'n_avg': n_avg,
1436
+ }
1437
+
1438
+ if not all_xbar_frames:
1439
+ sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
1440
+ _raise_no_replication('Xbar', sds, insufficient_strata, len(strata))
1441
+ chart_out = pd.concat(all_xbar_frames, ignore_index=True)
1442
+
1443
+ # Publish only strata a caller can actually focus() into.
1444
+ published_strata = [s for s in strata if s not in set(insufficient_strata)]
1445
+
1446
+ # Select output columns
1447
+ group_col = groupby_cols[0] if groupby_cols else None
1448
+ cols_to_keep = [stratify_col, group_col, 'xbar', 'center', 'lpl', 'upl', 'beyond_limits']
1449
+ cols_to_keep = [c for c in cols_to_keep if c is not None and c in chart_out.columns]
1450
+ chart_out = chart_out[cols_to_keep]
1451
+
1452
+ result['Xbar'] = {
1453
+ 'data': chart_out,
1454
+ 'statistics': chart_statistics,
1455
+ 'metadata': {
1456
+ 'chart_type': 'Xbar',
1457
+ 'value_col': 'xbar',
1458
+ 'center_col': 'center',
1459
+ 'stratified': True,
1460
+ 'stratify_col': stratify_col,
1461
+ 'stratify_by': list(stratify_by),
1462
+ 'n_sigma': self.request.n_sigma,
1463
+ 'n_mode': self.request.n_mode,
1464
+ 'insufficient_strata': insufficient_strata or None,
1465
+ },
1466
+ 'strata': published_strata,
1467
+ }
1468
+
1469
+ if _return_intermediates:
1470
+ result['_intermediates'] = {
1471
+ 'stratify_col': stratify_col,
1472
+ 'stratify_by': stratify_by,
1473
+ # Published, not raw: the companion S path indexes per_stratum[stratum]
1474
+ # directly, and a skipped stratum is absent from it — passing the raw list
1475
+ # here raised KeyError on any partially-replicated study.
1476
+ 'strata': published_strata,
1477
+ 'insufficient_strata': insufficient_strata,
1478
+ 'groupby_cols': groupby_cols,
1479
+ 'per_stratum': intermediates_per_stratum,
1480
+ 'is_stratified': True,
1481
+ }
1482
+
1483
+ return result
1484
+
1485
+ def _calculate_s_stratified(
1486
+ self,
1487
+ value_col: str,
1488
+ groupby_cols: list[str],
1489
+ stratify_by: list[str],
1490
+ _precomputed: dict | None = None,
1491
+ ) -> dict:
1492
+ """
1493
+ Stratified S chart: one chart per factor combination, time on x-axis.
1494
+
1495
+ Parameters
1496
+ ----------
1497
+ value_col : str
1498
+ Column to chart.
1499
+ groupby_cols : list[str]
1500
+ Columns to group by within each stratum.
1501
+ stratify_by : list[str]
1502
+ Columns defining strata.
1503
+ _precomputed : dict, optional
1504
+ Pre-computed intermediates from _calculate_xbar_stratified().
1505
+
1506
+ Returns
1507
+ -------
1508
+ dict
1509
+ Chart data with 'strata' key for stratified output.
1510
+ """
1511
+ spec = self.spec
1512
+
1513
+ if _precomputed is not None:
1514
+ stratify_col = _precomputed['stratify_col']
1515
+ # Already filtered to the strata Xbar could compute; per_stratum has no entry
1516
+ # for the rest, so iterating the raw list would KeyError.
1517
+ strata = _precomputed['strata']
1518
+ per_stratum = _precomputed['per_stratum']
1519
+ insufficient_strata = list(_precomputed.get('insufficient_strata') or [])
1520
+ else:
1521
+ df = self._source_frame().copy()
1522
+ if len(stratify_by) == 1:
1523
+ stratify_col = stratify_by[0]
1524
+ else:
1525
+ df['_stratify_key'] = encode_rsg_series(df, stratify_by)
1526
+ stratify_col = '_stratify_key'
1527
+ strata = df[stratify_col].unique().tolist()
1528
+ per_stratum = None
1529
+ insufficient_strata = []
1530
+
1531
+ all_strata = list(strata) + [s for s in insufficient_strata if s not in strata]
1532
+ all_s_frames = []
1533
+ chart_statistics = {}
1534
+
1535
+ for stratum in strata:
1536
+ if per_stratum is not None:
1537
+ # Use precomputed data from Xbar stratified
1538
+ inter = per_stratum[stratum]
1539
+ out = inter['out'].copy()
1540
+ # Always use Xbar's _S (R2-based for residuals) for CL and limits
1541
+ _S = inter['_S']
1542
+ # Drop s_value — S chart plots R2 std (same basis as CL/limits)
1543
+ if 's_value' in out.columns:
1544
+ out = out.drop(columns=['s_value'])
1545
+ n_to_use = inter['n_to_use']
1546
+ n_max = inter['n_max']
1547
+ n_avg = inter['n_avg']
1548
+ else:
1549
+ sdf = df[df[stratify_col] == stratum].copy()
1550
+ _limits_col = self._resolve_limits_column(value_col, sdf)
1551
+ agg_dict = {
1552
+ 's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
1553
+ 'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
1554
+ }
1555
+ out = sdf.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
1556
+ mask = out['n'].eq(1)
1557
+ out = out[~mask]
1558
+ if out.shape[0] == 0:
1559
+ insufficient_strata.append(stratum)
1560
+ continue
1561
+ out['N'] = out['n'].max()
1562
+ # CL from R2-based std — S chart plots same basis
1563
+ _S = out['s'].mean()
1564
+ n_to_use, n_max = self._determine_n_to_use(out)
1565
+ n_avg = None
1566
+ if self.request.n_mode == 'average':
1567
+ n_avg = out['n'].mean()
1568
+ out['N'] = n_avg
1569
+ n_to_use = 'N'
1570
+
1571
+ # Cast time column to string for categorical x-axis
1572
+ if groupby_cols:
1573
+ out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
1574
+
1575
+ out['center'] = _S
1576
+ out[['lpl', 'upl']] = calculate_limits_vectorized(
1577
+ 'S',
1578
+ sd=out['center'],
1579
+ N=out[n_to_use],
1580
+ sigma_multiplier=self.request.n_sigma,
1581
+ )
1582
+
1583
+ out = self._add_beyond_limits_flag(out, value_col='s')
1584
+ out = out.round(spec.round_to)
1585
+ out[stratify_col] = stratum
1586
+
1587
+ all_s_frames.append(out)
1588
+
1589
+ _has_fixed_n = n_to_use == 'N'
1590
+ chart_statistics[stratum] = {
1591
+ 'center': round(_S, spec.round_to),
1592
+ 'N': round(n_avg, spec.round_to) if n_avg is not None else (out['N'].iloc[0] if _has_fixed_n else None),
1593
+ 'lpl': round(out['lpl'].iloc[0], spec.round_to) if _has_fixed_n else None,
1594
+ 'upl': round(out['upl'].iloc[0], spec.round_to) if _has_fixed_n else None,
1595
+ **({'limits_vary': True} if not _has_fixed_n else {}),
1596
+ }
1597
+
1598
+ if not all_s_frames:
1599
+ # Previously fell through to pd.concat([]), which surfaces as a raw pandas
1600
+ # "No objects to concatenate". The ungrouped S path and the stratified Xbar
1601
+ # path both already raised here; this one was the gap.
1602
+ sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
1603
+ _raise_no_replication('S', sds, insufficient_strata, len(all_strata))
1604
+ chart_out = pd.concat(all_s_frames, ignore_index=True)
1605
+
1606
+ published_strata = [s for s in all_strata if s not in set(insufficient_strata)]
1607
+
1608
+ group_col = groupby_cols[0] if groupby_cols else None
1609
+ cols_to_keep = [stratify_col, group_col, 's', 'center', 'lpl', 'upl', 'beyond_limits']
1610
+ cols_to_keep = [c for c in cols_to_keep if c is not None and c in chart_out.columns]
1611
+ chart_out = chart_out[cols_to_keep]
1612
+
1613
+ return {
1614
+ 'S': {
1615
+ 'data': chart_out,
1616
+ 'statistics': chart_statistics,
1617
+ 'metadata': {
1618
+ 'chart_type': 'S',
1619
+ 'value_col': 's',
1620
+ 'center_col': 'center',
1621
+ 'stratified': True,
1622
+ 'stratify_col': stratify_col,
1623
+ 'stratify_by': list(stratify_by),
1624
+ 'n_sigma': self.request.n_sigma,
1625
+ 'n_mode': self.request.n_mode,
1626
+ 'insufficient_strata': insufficient_strata or None,
1627
+ },
1628
+ 'strata': published_strata,
1629
+ }
1630
+ }
1631
+
1632
+ def _calculate_s( # noqa: C901
1633
+ self, value_col: str | None = None, _precomputed: dict | None = None
1634
+ ) -> dict:
1635
+ """
1636
+ Calculate S (standard deviation) chart statistics.
1637
+
1638
+ Parameters
1639
+ ----------
1640
+ value_col : str, optional
1641
+ Column to use for calculations. Defaults to spec.response_var.
1642
+ Pass a residual column (R2, R5, etc.) for residual charts.
1643
+ _precomputed : dict, optional
1644
+ Pre-computed values from _calculate_xbar() to avoid redundant
1645
+ computation. Used internally by _calculate_xbar_s() for companion
1646
+ chart calculation. Keys: '_S', 'n_to_use', 'n_max', 'groupby_cols',
1647
+ 'group_col', 'out'.
1648
+
1649
+ Returns
1650
+ -------
1651
+ dict
1652
+ Chart data: {'S': {'data': df, 'statistics': dict, 'metadata': dict}}
1653
+
1654
+ Logic moved from calculate_statistics_S()
1655
+ """
1656
+ spec = self.spec
1657
+ if value_col is None:
1658
+ value_col = spec.response_var
1659
+
1660
+ # Use precomputed values if available (from _calculate_xbar_s)
1661
+ if _precomputed is not None:
1662
+ n_to_use = _precomputed['n_to_use']
1663
+ n_max = _precomputed['n_max']
1664
+ groupby_cols = _precomputed['groupby_cols']
1665
+ group_col = _precomputed['group_col']
1666
+ out = _precomputed['out'].copy()
1667
+ n_avg = _precomputed.get('n_avg')
1668
+ # Always use Xbar's _S (R2-based for residuals) for CL and limits
1669
+ _S = _precomputed['_S']
1670
+ # Drop s_value — S chart plots R2 std (same basis as CL/limits)
1671
+ if 's_value' in out.columns:
1672
+ out = out.drop(columns=['s_value'])
1673
+ else:
1674
+ # Independent calculation (existing logic)
1675
+ df = self._source_frame().copy()
1676
+
1677
+ # Resolve grouping based on `by` parameter
1678
+ groupby_cols, _, stratify_by = self._resolve_by_grouping(value_col)
1679
+
1680
+ # Stratified path: separate chart per factor combination
1681
+ if stratify_by:
1682
+ self._reject_calibration('stratified S')
1683
+ return self._calculate_s_stratified(
1684
+ value_col,
1685
+ groupby_cols,
1686
+ stratify_by,
1687
+ )
1688
+
1689
+ _limits_col = self._resolve_limits_column(value_col, df)
1690
+
1691
+ # Handle by=[] (collapse all) - single point chart
1692
+ if groupby_cols == []:
1693
+ _S = df[_limits_col].std()
1694
+ _N = len(df)
1695
+ out = pd.DataFrame({'group': ['All'], 's': [_S], 'n': [_N], 'N': [_N]})
1696
+ else:
1697
+ agg_dict = {
1698
+ 's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
1699
+ # Count on response_var (not value_col) to avoid NaN issues with residuals
1700
+ 'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
1701
+ }
1702
+ out = df.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
1703
+
1704
+ # remove groups with a single observation
1705
+ mask = out['n'].eq(1)
1706
+ out = out[~mask]
1707
+
1708
+ # Handle case where no subgroups have >1 observation
1709
+ if out.shape[0] == 0:
1710
+ sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
1711
+ _raise_no_replicated_subgroups('S', sds)
1712
+
1713
+ out['N'] = out['n'].max()
1714
+
1715
+ # CL from R2-based std — S chart plots same basis
1716
+ _S = out['s'].mean()
1717
+
1718
+ # Determine if subgroup sizes are constant or variable
1719
+ n_to_use, n_max = self._determine_n_to_use(out)
1720
+
1721
+ # Override n_to_use if n_mode="average"
1722
+ n_mode = self.request.n_mode
1723
+ n_avg = None
1724
+ if n_mode == 'average':
1725
+ n_avg = out['n'].mean()
1726
+ out['N'] = n_avg
1727
+ n_to_use = 'N'
1728
+
1729
+ # Determine group_col
1730
+ by = self.request.by
1731
+ if by is not None and len(by) > 0:
1732
+ if len(by) == 1:
1733
+ out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
1734
+ group_col = groupby_cols[0]
1735
+ else:
1736
+ out['subgroup'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
1737
+ group_col = 'subgroup'
1738
+ elif len(groupby_cols) > 1:
1739
+ out['group'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
1740
+ group_col = 'group'
1741
+ elif groupby_cols:
1742
+ group_col = groupby_cols[0]
1743
+ else:
1744
+ group_col = None
1745
+
1746
+ # Calculate S chart (common path for both precomputed and independent)
1747
+ out['groups'] = out['n'].count()
1748
+ n_mode = self.request.n_mode
1749
+
1750
+ if self.calibration is not None:
1751
+ # Standards-given S: center=c4(N)·sigma, B5(N)·sigma … B6(N)·sigma (per-row N).
1752
+ out[['center', 'lpl', 'upl']] = out.apply(
1753
+ lambda row: self._calibrated_row('S', N=row[n_to_use]),
1754
+ axis=1,
1755
+ )
1756
+ else:
1757
+ out['center'] = _S
1758
+ # Add limits columns
1759
+ out[['lpl', 'upl']] = calculate_limits_vectorized(
1760
+ 'S',
1761
+ sd=out['center'],
1762
+ N=out[n_to_use],
1763
+ sigma_multiplier=self.request.n_sigma,
1764
+ )
1765
+
1766
+ # Detect beyond limits signals
1767
+ out = self._add_beyond_limits_flag(out, value_col='s')
1768
+
1769
+ # Apply grouping column transformation if needed and not already done
1770
+ if _precomputed is None:
1771
+ # Already handled in independent path above
1772
+ pass
1773
+ else:
1774
+ # For precomputed path, apply group_col transformation
1775
+ by = self.request.by
1776
+ if by is not None and len(by) > 0 and 'subgroup' not in out.columns:
1777
+ if len(by) == 1:
1778
+ out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
1779
+ group_col = groupby_cols[0]
1780
+ else:
1781
+ out['subgroup'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
1782
+ group_col = 'subgroup'
1783
+ elif len(groupby_cols) > 1 and group_col == 'group' and 'group' not in out.columns:
1784
+ out['group'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
1785
+
1786
+ cols_to_keep = [group_col, 's', 'center', 'lpl', 'upl', 'beyond_limits']
1787
+ cols_to_keep = [c for c in cols_to_keep if c in out.columns]
1788
+ out = out[cols_to_keep]
1789
+ out = out.round(spec.round_to)
1790
+
1791
+ # Build statistics
1792
+ _S_stat = out['center'].iloc[0] if len(out) > 0 else None
1793
+
1794
+ statistics = {'center': round(_S_stat, spec.round_to) if _S_stat is not None else None}
1795
+ if n_to_use == 'N':
1796
+ statistics['N'] = round(n_avg, spec.round_to) if n_avg is not None else n_max
1797
+ statistics['upl'] = out['upl'].max()
1798
+ statistics['lpl'] = out['lpl'].max()
1799
+ else:
1800
+ statistics['N'] = None
1801
+ statistics['lpl'] = None
1802
+ statistics['upl'] = None
1803
+ statistics['limits_vary'] = True
1804
+
1805
+ return {
1806
+ 'S': {
1807
+ 'data': out,
1808
+ 'statistics': statistics,
1809
+ 'metadata': {
1810
+ 'chart_type': 'S',
1811
+ 'value_col': 's',
1812
+ 'center_col': 'center',
1813
+ 'n_sigma': self.request.n_sigma,
1814
+ 'n_mode': n_mode,
1815
+ **({'n_avg': round(n_avg, spec.round_to)} if n_avg is not None else {}),
1816
+ },
1817
+ }
1818
+ }
1819
+
1820
+ def _calculate_xbar_s(self, value_col: str | None = None) -> dict[str, ChartPayload]:
1821
+ """
1822
+ Calculate Xbar and S charts together (Bishop methodology).
1823
+
1824
+ This method ensures that when companion=True, both charts are calculated
1825
+ efficiently using shared intermediate values. The S chart uses the
1826
+ same aggregated data computed for Xbar, avoiding redundant calculation.
1827
+
1828
+ Parameters
1829
+ ----------
1830
+ value_col : str, optional
1831
+ Column to use for calculations. Defaults to spec.response_var.
1832
+
1833
+ Returns
1834
+ -------
1835
+ dict
1836
+ Combined chart data: {'Xbar': {...}, 'S': {...}}
1837
+ """
1838
+ # Calculate Xbar with intermediates for S reuse
1839
+ xbar_result = self._calculate_xbar(value_col, _return_intermediates=True)
1840
+
1841
+ # Extract intermediates and remove from result
1842
+ intermediates = xbar_result.pop('_intermediates')
1843
+
1844
+ # Stratified companion path
1845
+ if intermediates.get('is_stratified'):
1846
+ s_result = self._calculate_s_stratified(
1847
+ value_col,
1848
+ intermediates['groupby_cols'],
1849
+ intermediates['stratify_by'],
1850
+ _precomputed=intermediates,
1851
+ )
1852
+ return {**xbar_result, **s_result}
1853
+
1854
+ # Calculate S using precomputed values
1855
+ s_result = self._calculate_s(value_col, _precomputed=intermediates)
1856
+
1857
+ # Combine results
1858
+ return {**xbar_result, **s_result}
1859
+
1860
+ @staticmethod
1861
+ def _resolve_mr_source_column(value_col: str) -> str:
1862
+ """Map recentered column (RCR*) to its non-recentered residual (R*).
1863
+
1864
+ Moving ranges must be computed from non-recentered residuals to avoid
1865
+ inflating mR with structural jumps between cells. Returns value_col
1866
+ unchanged if it is not a recentered column.
1867
+ """
1868
+ if value_col.startswith('RCR'):
1869
+ return value_col[2:] # RCR3 -> R3, RCR6 -> R6, etc.
1870
+ return value_col
1871
+
1872
+ def _calculate_mr_chart(
1873
+ self,
1874
+ mr_spec: _MRChartSpec,
1875
+ value_col: str | None = None,
1876
+ _return_intermediates: bool = False,
1877
+ _precomputed: dict | None = None,
1878
+ ) -> dict:
1879
+ """
1880
+ Shared pipeline for MR-family charts (X and mR).
1881
+
1882
+ The X and mR charts share >85% of their calculation logic. The behavioral
1883
+ differences are encoded in ``mr_spec`` — no boolean flags needed.
1884
+
1885
+ Parameters
1886
+ ----------
1887
+ mr_spec : _MRChartSpec
1888
+ Chart behavior specification (_XMR_SPEC or _R_SPEC).
1889
+ value_col : str, optional
1890
+ Column to use for calculations. Defaults to spec.response_var.
1891
+ _return_intermediates : bool, optional
1892
+ If True, includes '_intermediates' key for companion-mode reuse.
1893
+ _precomputed : dict, optional
1894
+ Pre-computed intermediates from a prior X call (companion mode).
1895
+ When provided, skips data preparation and uses these values directly.
1896
+
1897
+ Returns
1898
+ -------
1899
+ dict
1900
+ Chart data: {chart_type: {'data': df, 'statistics': dict, 'metadata': dict}}
1901
+ """
1902
+ spec = self.spec
1903
+ if value_col is None:
1904
+ value_col = spec.response_var
1905
+
1906
+ logger.debug('In calculate statistics %s', mr_spec.chart_type)
1907
+
1908
+ # --- Determine stratification (shared logic) ---
1909
+ stratify_by, collapsed_factors = self._resolve_xmr_stratification()
1910
+ is_stratified = len(stratify_by) > 0
1911
+
1912
+ # --- Resolve plot column ---
1913
+ plot_col = value_col if mr_spec.plot_col == 'raw' else 'mr'
1914
+
1915
+ # --- Resolve mR source column ---
1916
+ # When recentered (value_col is RCR*), compute mR from the non-recentered
1917
+ # residual (R*) to avoid inflating moving ranges with structural jumps.
1918
+ mr_source_col = self._resolve_mr_source_column(value_col)
1919
+
1920
+ if is_stratified:
1921
+ self._reject_calibration('stratified X/mR')
1922
+ return self._calculate_mr_chart_stratified(
1923
+ mr_spec,
1924
+ value_col,
1925
+ plot_col,
1926
+ mr_source_col,
1927
+ stratify_by,
1928
+ collapsed_factors,
1929
+ _return_intermediates,
1930
+ _precomputed,
1931
+ phased=self.request.phased,
1932
+ )
1933
+ else:
1934
+ return self._calculate_mr_chart_ungrouped(
1935
+ mr_spec,
1936
+ value_col,
1937
+ plot_col,
1938
+ mr_source_col,
1939
+ collapsed_factors,
1940
+ _return_intermediates,
1941
+ _precomputed,
1942
+ phased=self.request.phased,
1943
+ )
1944
+
1945
+ def _calculate_mr_chart_from_precomputed(
1946
+ self,
1947
+ mr_spec: _MRChartSpec,
1948
+ plot_col: str,
1949
+ precomputed: dict,
1950
+ ) -> dict:
1951
+ """Build an MR-family chart reusing intermediates from a companion X call."""
1952
+ spec = self.spec
1953
+ out = precomputed['out'].copy()
1954
+ stratify_col = precomputed['stratify_col']
1955
+ stratify_by = precomputed['stratify_by']
1956
+ strata = precomputed['strata']
1957
+ xmr_lane_boundaries = precomputed['lane_boundaries']
1958
+ extra_cols = precomputed['extra_cols']
1959
+
1960
+ # --- Phased precomputed path ---
1961
+ if precomputed.get('phased'):
1962
+ phase_stats = precomputed['phase_stats']
1963
+
1964
+ # Recompute R-specific limits per phase within each stratum
1965
+ r_phase_stats = phase_stats[[stratify_col, '_phase_id', 'mR']].copy()
1966
+ r_lims = r_phase_stats.apply(
1967
+ lambda row: calculate_limits(
1968
+ mean=0,
1969
+ sd=0,
1970
+ N=0,
1971
+ mR=row['mR'],
1972
+ limits_type=mr_spec.limits_type,
1973
+ round_to=spec.round_to,
1974
+ ),
1975
+ axis=1,
1976
+ )
1977
+ r_phase_stats = _join_limits(r_phase_stats, r_lims)
1978
+ r_phase_stats['center'] = r_phase_stats['mR']
1979
+
1980
+ # Replace X limits with mR limits
1981
+ out = out.drop(columns=['center', 'lpl', 'upl', 'beyond_limits'], errors='ignore')
1982
+ out = out.merge(
1983
+ r_phase_stats[[stratify_col, '_phase_id', 'center', 'lpl', 'upl']],
1984
+ on=[stratify_col, '_phase_id'],
1985
+ how='left',
1986
+ validate='many_to_one',
1987
+ )
1988
+
1989
+ # Re-detect beyond limits
1990
+ if plot_col not in out.columns:
1991
+ raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
1992
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
1993
+
1994
+ # Format output
1995
+ chart_out = self._build_output_columns(
1996
+ df=out,
1997
+ value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
1998
+ )
1999
+
2000
+ chart_statistics = {
2001
+ stratum: {
2002
+ 'N': None,
2003
+ 'center': None,
2004
+ 'lpl': None,
2005
+ 'upl': None,
2006
+ 'limits_vary': True,
2007
+ }
2008
+ for stratum in strata
2009
+ }
2010
+
2011
+ # Phased R: no lane boundary offset (no rows dropped)
2012
+ return _build_stratified_mr_payload(
2013
+ mr_spec=mr_spec,
2014
+ chart_out=chart_out,
2015
+ chart_statistics=chart_statistics,
2016
+ plot_col=plot_col,
2017
+ stratify_col=stratify_col,
2018
+ stratify_by=stratify_by,
2019
+ strata=strata,
2020
+ lane_boundaries=xmr_lane_boundaries,
2021
+ phased=True,
2022
+ )
2023
+
2024
+ # --- Non-phased precomputed path ---
2025
+ grouped = precomputed['grouped'].copy()
2026
+
2027
+ # For mR chart: compute mR-specific limits from X's grouped data
2028
+ # grouped has columns: stratify_col, center (=mean), mR, lpl, upl (X limits)
2029
+ r_grouped = grouped.rename(columns={'center': 'xmr_center', 'mR': 'center'})
2030
+
2031
+ r_lims = r_grouped.apply(
2032
+ lambda row: calculate_limits(
2033
+ mean=0,
2034
+ sd=0,
2035
+ N=0,
2036
+ mR=row['center'],
2037
+ limits_type=mr_spec.limits_type,
2038
+ round_to=spec.round_to,
2039
+ ),
2040
+ axis=1,
2041
+ )
2042
+
2043
+ r_grouped = _join_limits(r_grouped, r_lims, rsuffix='_r')
2044
+
2045
+ # Resolve lpl/upl column names (may have suffix if X lpl/upl exist)
2046
+ if 'lpl_r' in r_grouped.columns:
2047
+ r_grouped = r_grouped.rename(columns={'lpl_r': 'r_lpl', 'upl_r': 'r_upl'})
2048
+ r_lpl_col, r_upl_col = 'r_lpl', 'r_upl'
2049
+ else:
2050
+ r_lpl_col, r_upl_col = 'lpl', 'upl'
2051
+
2052
+ # Merge mR limits to the data (drop X-specific columns first)
2053
+ r_out = out.drop(columns=['center', 'lpl', 'upl', 'beyond_limits'], errors='ignore')
2054
+ r_merge_data = r_grouped[[stratify_col, 'center', r_lpl_col, r_upl_col]].rename(
2055
+ columns={r_lpl_col: 'lpl', r_upl_col: 'upl'}
2056
+ )
2057
+ r_out = r_out.merge(r_merge_data, on=stratify_col, how='left', validate='many_to_one')
2058
+
2059
+ # R chart drops first observation per stratum (NaN moving range)
2060
+ if mr_spec.drops_first_mr:
2061
+ r_out = r_out.dropna(subset=['mr'])
2062
+
2063
+ # Detect beyond limits
2064
+ r_out = self._add_beyond_limits_flag(r_out, value_col=plot_col)
2065
+
2066
+ # Build statistics
2067
+ chart_statistics = {}
2068
+ for stratum in strata:
2069
+ row = r_grouped[r_grouped[stratify_col] == stratum].iloc[0]
2070
+ chart_statistics[stratum] = {
2071
+ 'N': _mr_family_subgroup_n(mr_spec.chart_type),
2072
+ 'center': round(row['center'], spec.round_to),
2073
+ 'lpl': round(row[r_lpl_col], spec.round_to),
2074
+ 'upl': round(row[r_upl_col], spec.round_to),
2075
+ }
2076
+
2077
+ # Format output
2078
+ chart_out = self._build_output_columns(
2079
+ df=r_out,
2080
+ value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2081
+ )
2082
+
2083
+ lane_boundaries = _shift_lane_boundaries(xmr_lane_boundaries, mr_spec.lane_boundary_offset)
2084
+ insufficient_strata, _ = _split_strata_by_sufficiency(r_out, stratify_col, strata)
2085
+
2086
+ return _build_stratified_mr_payload(
2087
+ mr_spec=mr_spec,
2088
+ chart_out=chart_out,
2089
+ chart_statistics=chart_statistics,
2090
+ plot_col=plot_col,
2091
+ stratify_col=stratify_col,
2092
+ stratify_by=stratify_by,
2093
+ strata=strata,
2094
+ lane_boundaries=lane_boundaries,
2095
+ insufficient_strata=insufficient_strata,
2096
+ )
2097
+
2098
+ def _calculate_mr_chart_stratified( # noqa: C901
2099
+ self,
2100
+ mr_spec: _MRChartSpec,
2101
+ value_col: str,
2102
+ plot_col: str,
2103
+ mr_source_col: str,
2104
+ stratify_by: list[str],
2105
+ collapsed_factors: list[str],
2106
+ _return_intermediates: bool,
2107
+ _precomputed: dict | None,
2108
+ phased: bool = False,
2109
+ ) -> dict:
2110
+ """Stratified path for _calculate_mr_chart."""
2111
+ spec = self.spec
2112
+
2113
+ if _precomputed is not None and _precomputed['is_stratified']:
2114
+ return self._calculate_mr_chart_from_precomputed(
2115
+ mr_spec,
2116
+ plot_col,
2117
+ _precomputed,
2118
+ )
2119
+
2120
+ # --- Independent calculation (no precomputed values) ---
2121
+ result = {}
2122
+ out = self._source_frame().copy()
2123
+ logger.debug('Dataframe has columns: %s', out.columns.to_list())
2124
+
2125
+ # Determine stratification column
2126
+ if stratify_by == [spec.rsg_var_name]:
2127
+ stratify_col = spec.rsg_var_name
2128
+ elif len(stratify_by) == 1:
2129
+ stratify_col = stratify_by[0]
2130
+ else:
2131
+ out['_stratify_key'] = encode_rsg_series(out, stratify_by)
2132
+ stratify_col = '_stratify_key'
2133
+
2134
+ # Canonical sort
2135
+ out = out.sort_values('sort_key', kind='stable')
2136
+
2137
+ # Strata list (needed for both phased and non-phased paths)
2138
+ strata = out[stratify_col].unique().tolist()
2139
+
2140
+ # === Phased limits path (per-phase limits within each stratum) ===
2141
+ if phased and collapsed_factors:
2142
+ # 1. Build phase key from collapsed factors only (vectorized)
2143
+ if len(collapsed_factors) == 1:
2144
+ out['_phase_key'] = out[collapsed_factors[0]]
2145
+ else:
2146
+ out['_phase_key'] = list(map(tuple, out[collapsed_factors].to_numpy()))
2147
+
2148
+ # 2. Assign within-stratum phase IDs via transitions of _phase_key
2149
+ out['_phase_id'] = (
2150
+ out.groupby(stratify_col, sort=False)['_phase_key']
2151
+ .transform(lambda s: (s != s.shift()).cumsum())
2152
+ .astype('int64')
2153
+ )
2154
+
2155
+ # 3. Per-phase mR (reset at phase boundaries within each stratum)
2156
+ out['mr'] = out.groupby([stratify_col, '_phase_id'], sort=False)[mr_source_col].diff().abs()
2157
+
2158
+ # 4. Per-phase aggregation
2159
+ phase_stats = (
2160
+ out.groupby([stratify_col, '_phase_id'], sort=False)
2161
+ .agg(_n=(value_col, 'size'), _mean=(value_col, 'mean'), mR=('mr', 'mean'))
2162
+ .reset_index()
2163
+ )
2164
+ phase_stats['mR'] = phase_stats['mR'].fillna(0.0)
2165
+
2166
+ # 5. Compute per-phase limits
2167
+ if mr_spec.center_source == 'mean':
2168
+ phase_stats['center'] = phase_stats['_mean']
2169
+ phase_lims = phase_stats.apply(
2170
+ lambda row: calculate_limits(
2171
+ mean=row['_mean'],
2172
+ sd=0,
2173
+ N=0,
2174
+ mR=row['mR'],
2175
+ limits_type=mr_spec.limits_type,
2176
+ round_to=spec.round_to,
2177
+ ),
2178
+ axis=1,
2179
+ )
2180
+ else: # center_source == 'mR' (R chart)
2181
+ phase_stats['center'] = phase_stats['mR']
2182
+ phase_lims = phase_stats.apply(
2183
+ lambda row: calculate_limits(
2184
+ mean=0,
2185
+ sd=0,
2186
+ N=0,
2187
+ mR=row['mR'],
2188
+ limits_type=mr_spec.limits_type,
2189
+ round_to=spec.round_to,
2190
+ ),
2191
+ axis=1,
2192
+ )
2193
+ phase_stats = _join_limits(phase_stats, phase_lims)
2194
+
2195
+ # 6. Merge per-phase limits back to row-level data
2196
+ out = out.merge(
2197
+ phase_stats[[stratify_col, '_phase_id', 'center', 'mR', 'lpl', 'upl']],
2198
+ on=[stratify_col, '_phase_id'],
2199
+ how='left',
2200
+ validate='many_to_one',
2201
+ )
2202
+
2203
+ # 7. Phased path: NEVER drop rows. NaN mR at phase boundaries preserved.
2204
+
2205
+ # 8. Signal detection (per-row, uses row-level lpl/upl)
2206
+ if plot_col not in out.columns:
2207
+ raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
2208
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2209
+
2210
+ # 9. Lane boundaries per stratum
2211
+ lane_boundaries = {}
2212
+ if collapsed_factors:
2213
+ for stratum in strata:
2214
+ stratum_data = out[out[stratify_col] == stratum].reset_index(drop=True)
2215
+ boundaries = self._calculate_lane_boundaries(stratum_data, collapsed_factors)
2216
+ if boundaries:
2217
+ lane_boundaries[stratum] = boundaries
2218
+
2219
+ # 10. Statistics per stratum: limits vary per phase
2220
+ chart_statistics = {
2221
+ stratum: {
2222
+ 'N': None,
2223
+ 'center': None,
2224
+ 'lpl': None,
2225
+ 'upl': None,
2226
+ 'limits_vary': True,
2227
+ }
2228
+ for stratum in strata
2229
+ }
2230
+
2231
+ # 11. Track single-point phases
2232
+ n_single_point = int((phase_stats['_n'] < 2).sum())
2233
+
2234
+ # Format output
2235
+ extra_cols = [c for c in stratify_by if c != spec.rsg_var_name and c != spec.time_var]
2236
+ chart_out = self._build_output_columns(
2237
+ df=out,
2238
+ value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2239
+ )
2240
+
2241
+ result.update(_build_stratified_mr_payload(
2242
+ mr_spec=mr_spec,
2243
+ chart_out=chart_out,
2244
+ chart_statistics=chart_statistics,
2245
+ plot_col=plot_col,
2246
+ stratify_col=stratify_col,
2247
+ stratify_by=stratify_by,
2248
+ strata=strata,
2249
+ lane_boundaries=lane_boundaries,
2250
+ phased=True,
2251
+ single_point_phases=n_single_point,
2252
+ ))
2253
+
2254
+ if _return_intermediates:
2255
+ result['_intermediates'] = {
2256
+ 'out': out,
2257
+ 'phase_stats': phase_stats,
2258
+ 'stratify_col': stratify_col,
2259
+ 'stratify_by': stratify_by,
2260
+ 'strata': strata,
2261
+ 'lane_boundaries': lane_boundaries,
2262
+ 'collapsed_factors': collapsed_factors,
2263
+ 'extra_cols': extra_cols,
2264
+ 'is_stratified': True,
2265
+ 'phased': True,
2266
+ }
2267
+
2268
+ return result
2269
+
2270
+ # === Global limits path (unchanged) ===
2271
+
2272
+ # Moving range per stratum
2273
+ out['mr'] = out.groupby(stratify_col, sort=False, observed=True)[mr_source_col].diff().abs()
2274
+
2275
+ # Aggregate per stratum
2276
+ agg = {}
2277
+ if value_col in out.columns:
2278
+ agg['mean'] = (value_col, 'mean')
2279
+ if 'mr' in out.columns:
2280
+ agg['mR'] = ('mr', 'mean')
2281
+
2282
+ if not agg:
2283
+ raise RuntimeError(
2284
+ f'{mr_spec.chart_type} aggregation spec is empty. '
2285
+ f'Have columns: {out.columns.tolist()}, value_col={value_col!r}'
2286
+ )
2287
+
2288
+ grouped = out.groupby(stratify_col, sort=False, observed=True).agg(**agg).reset_index()
2289
+
2290
+ # Compute limits per group
2291
+ # For X: center=mean, limits based on mean ± E2*mR
2292
+ # For mR: center=mR, limits based on 0 to D4*mR
2293
+ if mr_spec.center_source == 'mean':
2294
+ lims = grouped.apply(
2295
+ lambda row: calculate_limits(
2296
+ mean=row['mean'],
2297
+ sd=0,
2298
+ N=0,
2299
+ mR=(row['mR'] if pd.notna(row['mR']) else 0.0),
2300
+ limits_type=mr_spec.limits_type,
2301
+ round_to=spec.round_to,
2302
+ ),
2303
+ axis=1,
2304
+ )
2305
+ else: # center_source == 'mR'
2306
+ lims = grouped.apply(
2307
+ lambda row: calculate_limits(
2308
+ mean=0,
2309
+ sd=0,
2310
+ N=0,
2311
+ mR=row['mR'],
2312
+ limits_type=mr_spec.limits_type,
2313
+ round_to=spec.round_to,
2314
+ ),
2315
+ axis=1,
2316
+ )
2317
+
2318
+ # Normalize/join lpl/upl
2319
+ grouped = _join_limits(grouped, lims)
2320
+
2321
+ # Set center column based on chart type
2322
+ if mr_spec.center_source == 'mean':
2323
+ grouped = grouped.rename(columns={'mean': 'center'})
2324
+ else:
2325
+ grouped['center'] = grouped['mR']
2326
+
2327
+ # Merge limits back to row-level data
2328
+ out = out.merge(
2329
+ grouped[[stratify_col, 'center', 'mR', 'lpl', 'upl']],
2330
+ on=stratify_col,
2331
+ how='left',
2332
+ validate='many_to_one',
2333
+ )
2334
+
2335
+ # R chart: drop first observation per stratum (NaN moving range)
2336
+ if mr_spec.drops_first_mr:
2337
+ out = out.dropna(subset=['mr'])
2338
+
2339
+ # Detect beyond limits
2340
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2341
+
2342
+ # Use strata from grouped for non-phased path (preserves original order)
2343
+ strata = grouped[stratify_col].tolist()
2344
+
2345
+ # Lane boundaries per stratum
2346
+ lane_boundaries = {}
2347
+ if collapsed_factors:
2348
+ for stratum in strata:
2349
+ stratum_data = out[out[stratify_col] == stratum].reset_index(drop=True)
2350
+ boundaries = self._calculate_lane_boundaries(stratum_data, collapsed_factors)
2351
+ if boundaries:
2352
+ lane_boundaries[stratum] = boundaries
2353
+
2354
+ # Build statistics nested by stratum
2355
+ chart_statistics = {}
2356
+ for stratum in strata:
2357
+ row = grouped[grouped[stratify_col] == stratum].iloc[0]
2358
+ chart_statistics[stratum] = {
2359
+ 'N': _mr_family_subgroup_n(mr_spec.chart_type),
2360
+ 'center': round(row['center'], spec.round_to),
2361
+ 'lpl': round(row['lpl'], spec.round_to),
2362
+ 'upl': round(row['upl'], spec.round_to),
2363
+ }
2364
+
2365
+ # Format output
2366
+ extra_cols = [c for c in stratify_by if c != spec.rsg_var_name and c != spec.time_var]
2367
+ chart_out = self._build_output_columns(
2368
+ df=out,
2369
+ value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2370
+ )
2371
+
2372
+ insufficient_strata, _ = _split_strata_by_sufficiency(out, stratify_col, strata)
2373
+
2374
+ result.update(_build_stratified_mr_payload(
2375
+ mr_spec=mr_spec,
2376
+ chart_out=chart_out,
2377
+ chart_statistics=chart_statistics,
2378
+ plot_col=plot_col,
2379
+ stratify_col=stratify_col,
2380
+ stratify_by=stratify_by,
2381
+ strata=strata,
2382
+ lane_boundaries=lane_boundaries,
2383
+ insufficient_strata=insufficient_strata,
2384
+ ))
2385
+
2386
+ # Optionally include intermediates for companion calculation
2387
+ if _return_intermediates:
2388
+ result['_intermediates'] = {
2389
+ 'out': out,
2390
+ 'grouped': grouped,
2391
+ 'stratify_col': stratify_col,
2392
+ 'stratify_by': stratify_by,
2393
+ 'strata': strata,
2394
+ 'lane_boundaries': lane_boundaries,
2395
+ 'collapsed_factors': collapsed_factors,
2396
+ 'extra_cols': extra_cols,
2397
+ 'is_stratified': True,
2398
+ }
2399
+
2400
+ return result
2401
+
2402
+ def _calculate_mr_chart_ungrouped( # noqa: C901
2403
+ self,
2404
+ mr_spec: _MRChartSpec,
2405
+ value_col: str,
2406
+ plot_col: str,
2407
+ mr_source_col: str,
2408
+ collapsed_factors: list[str],
2409
+ _return_intermediates: bool,
2410
+ _precomputed: dict | None,
2411
+ phased: bool = False,
2412
+ ) -> dict:
2413
+ """Ungrouped (single-stream) path for _calculate_mr_chart."""
2414
+ spec = self.spec
2415
+ result = {}
2416
+
2417
+ if _precomputed is not None and not _precomputed['is_stratified']:
2418
+ # --- Phased precomputed path (for companion R chart) ---
2419
+ if _precomputed.get('phased'):
2420
+ out = _precomputed['out'].copy()
2421
+ phase_stats = _precomputed['phase_stats']
2422
+ xmr_lane_boundaries = _precomputed['lane_boundaries']
2423
+
2424
+ # Recompute R-specific limits per phase
2425
+ r_phase_stats = phase_stats[['_phase_id', 'mR']].copy()
2426
+ r_lims = r_phase_stats.apply(
2427
+ lambda row: calculate_limits(
2428
+ mean=0,
2429
+ sd=0,
2430
+ N=0,
2431
+ mR=row['mR'],
2432
+ limits_type=mr_spec.limits_type,
2433
+ round_to=spec.round_to,
2434
+ ),
2435
+ axis=1,
2436
+ )
2437
+ r_phase_stats = _join_limits(r_phase_stats, r_lims)
2438
+ r_phase_stats['center'] = r_phase_stats['mR']
2439
+
2440
+ # Replace X limits with mR limits
2441
+ out = out.drop(columns=['center', 'lpl', 'upl'], errors='ignore')
2442
+ out = out.merge(
2443
+ r_phase_stats[['_phase_id', 'center', 'lpl', 'upl']],
2444
+ on='_phase_id',
2445
+ how='left',
2446
+ validate='many_to_one',
2447
+ )
2448
+
2449
+ # Re-detect beyond limits
2450
+ out = out.drop(columns=['beyond_limits'], errors='ignore')
2451
+ if plot_col not in out.columns:
2452
+ raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
2453
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2454
+
2455
+ chart_out = self._build_output_columns(
2456
+ df=out,
2457
+ value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2458
+ )
2459
+
2460
+ chart_statistics = {
2461
+ 'N': None,
2462
+ 'center': None,
2463
+ 'lpl': None,
2464
+ 'upl': None,
2465
+ 'limits_vary': True,
2466
+ }
2467
+
2468
+ # Phased R: no lane boundary offset needed (no rows dropped)
2469
+ result.update(_build_ungrouped_mr_payload(
2470
+ mr_spec=mr_spec,
2471
+ chart_out=chart_out,
2472
+ chart_statistics=chart_statistics,
2473
+ plot_col=plot_col,
2474
+ lane_boundaries=xmr_lane_boundaries,
2475
+ phased=True,
2476
+ ))
2477
+ return result
2478
+
2479
+ # --- Existing non-phased precomputed path ---
2480
+ # Reuse intermediates from companion X call
2481
+ out = _precomputed['out'].copy()
2482
+ mR = _precomputed['mR']
2483
+ xmr_lane_boundaries = _precomputed['lane_boundaries']
2484
+
2485
+ # R chart: drop first observation (NaN moving range)
2486
+ if mr_spec.drops_first_mr:
2487
+ out = out.dropna(subset=['mr'])
2488
+
2489
+ # Compute R-specific limits
2490
+ if self.calibration is not None:
2491
+ center_r, r_lims = self._calibrated_scalar(mr_spec.limits_type)
2492
+ else:
2493
+ center_r = mR
2494
+ r_lims = calculate_limits(
2495
+ mean=0,
2496
+ sd=0,
2497
+ N=0,
2498
+ mR=mR,
2499
+ limits_type=mr_spec.limits_type,
2500
+ round_to=spec.round_to,
2501
+ )
2502
+ out['center'] = center_r
2503
+ out['lpl'] = r_lims['lpl']
2504
+ out['upl'] = r_lims['upl']
2505
+
2506
+ # Re-detect beyond limits for R chart
2507
+ out = out.drop(columns=['beyond_limits'], errors='ignore')
2508
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2509
+
2510
+ # Format output
2511
+ chart_out = self._build_output_columns(
2512
+ df=out,
2513
+ value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2514
+ )
2515
+
2516
+ chart_statistics = {
2517
+ 'N': _mr_family_subgroup_n(mr_spec.chart_type),
2518
+ 'center': round(center_r, spec.round_to),
2519
+ 'lpl': round(r_lims['lpl'], spec.round_to),
2520
+ 'upl': round(r_lims['upl'], spec.round_to),
2521
+ }
2522
+
2523
+ r_lane_boundaries = _shift_lane_boundaries(
2524
+ xmr_lane_boundaries, mr_spec.lane_boundary_offset
2525
+ )
2526
+
2527
+ result.update(_build_ungrouped_mr_payload(
2528
+ mr_spec=mr_spec,
2529
+ chart_out=chart_out,
2530
+ chart_statistics=chart_statistics,
2531
+ plot_col=plot_col,
2532
+ lane_boundaries=r_lane_boundaries,
2533
+ ))
2534
+ return result
2535
+
2536
+ # --- Independent calculation ---
2537
+ out = self._source_frame().copy()
2538
+ out = out.sort_values('sort_key', kind='stable')
2539
+ out = out.reset_index(drop=True)
2540
+
2541
+ # Lane boundaries before any row drops
2542
+ lane_boundaries = None
2543
+ if collapsed_factors:
2544
+ lane_boundaries = self._calculate_lane_boundaries(out, collapsed_factors)
2545
+
2546
+ # === Phased limits path ===
2547
+ if phased and collapsed_factors:
2548
+ self._reject_calibration('phased X/mR')
2549
+ # 1. Assign phase_id: contiguous runs of the same rsg_key
2550
+ out['_phase_id'] = (out['rsg_key'] != out['rsg_key'].shift()).cumsum().astype('int64')
2551
+
2552
+ # 2. Per-phase moving range (reset at boundaries)
2553
+ out['mr'] = out.groupby('_phase_id', sort=False)[mr_source_col].diff().abs()
2554
+
2555
+ # 3. Per-phase aggregation
2556
+ phase_stats = (
2557
+ out.groupby('_phase_id', sort=False)
2558
+ .agg(
2559
+ _n=(value_col, 'size'),
2560
+ _mean=(value_col, 'mean'),
2561
+ mR=('mr', 'mean'),
2562
+ )
2563
+ .reset_index()
2564
+ )
2565
+
2566
+ # 4. Handle single-point phases (mR=NaN -> use 0)
2567
+ phase_stats['mR'] = phase_stats['mR'].fillna(0.0)
2568
+
2569
+ # 5. Compute per-phase limits
2570
+ if mr_spec.center_source == 'mean':
2571
+ phase_stats['center'] = phase_stats['_mean']
2572
+ phase_lims = phase_stats.apply(
2573
+ lambda row: calculate_limits(
2574
+ mean=row['_mean'],
2575
+ sd=0,
2576
+ N=0,
2577
+ mR=row['mR'],
2578
+ limits_type=mr_spec.limits_type,
2579
+ round_to=spec.round_to,
2580
+ ),
2581
+ axis=1,
2582
+ )
2583
+ else: # center_source == 'mR' (R chart)
2584
+ phase_stats['center'] = phase_stats['mR']
2585
+ phase_lims = phase_stats.apply(
2586
+ lambda row: calculate_limits(
2587
+ mean=0,
2588
+ sd=0,
2589
+ N=0,
2590
+ mR=row['mR'],
2591
+ limits_type=mr_spec.limits_type,
2592
+ round_to=spec.round_to,
2593
+ ),
2594
+ axis=1,
2595
+ )
2596
+ phase_stats = _join_limits(phase_stats, phase_lims)
2597
+
2598
+ # 6. Merge per-phase limits back to row-level data
2599
+ out = out.merge(
2600
+ phase_stats[['_phase_id', 'center', 'mR', 'lpl', 'upl']],
2601
+ on='_phase_id',
2602
+ how='left',
2603
+ validate='many_to_one',
2604
+ )
2605
+
2606
+ # 7. Phased path: NEVER drop rows. NaN mR values at phase boundaries
2607
+ # stay in the DataFrame. Plotly skips NaN naturally.
2608
+
2609
+ # 8. Signal detection (per-row, uses row-level lpl/upl)
2610
+ if plot_col not in out.columns:
2611
+ raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
2612
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2613
+
2614
+ # 9. Format output
2615
+ chart_out = self._build_output_columns(
2616
+ df=out,
2617
+ value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2618
+ )
2619
+
2620
+ # 10. Statistics: limits_vary=True signals stepped rendering
2621
+ chart_statistics = {
2622
+ 'N': None,
2623
+ 'center': None,
2624
+ 'lpl': None,
2625
+ 'upl': None,
2626
+ 'limits_vary': True,
2627
+ }
2628
+
2629
+ # 11. Track single-point phases for visibility
2630
+ n_single_point = int((phase_stats['_n'] < 2).sum())
2631
+
2632
+ result.update(_build_ungrouped_mr_payload(
2633
+ mr_spec=mr_spec,
2634
+ chart_out=chart_out,
2635
+ chart_statistics=chart_statistics,
2636
+ plot_col=plot_col,
2637
+ lane_boundaries=lane_boundaries,
2638
+ phased=True,
2639
+ single_point_phases=n_single_point,
2640
+ ))
2641
+
2642
+ if _return_intermediates:
2643
+ result['_intermediates'] = {
2644
+ 'out': out,
2645
+ 'mR': None,
2646
+ 'lane_boundaries': lane_boundaries,
2647
+ 'is_stratified': False,
2648
+ 'phased': True,
2649
+ 'phase_stats': phase_stats,
2650
+ }
2651
+
2652
+ return result
2653
+
2654
+ # === Global limits path (unchanged) ===
2655
+ # Moving range
2656
+ out['mr'] = out[mr_source_col].diff().abs()
2657
+ mR = out['mr'].mean()
2658
+ mean_ = out[value_col].mean()
2659
+
2660
+ # R chart: drop first observation
2661
+ if mr_spec.drops_first_mr:
2662
+ out = out.dropna(subset=['mr'])
2663
+
2664
+ # Compute limits
2665
+ if self.calibration is not None:
2666
+ # Standards-given: X -> mean ± 3·sigma; mR -> 0 … D4·d2·sigma.
2667
+ center_val, lims = self._calibrated_scalar(mr_spec.limits_type)
2668
+ elif mr_spec.center_source == 'mean':
2669
+ center_val = mean_
2670
+ lims = calculate_limits(
2671
+ mean=mean_,
2672
+ sd=0,
2673
+ N=0,
2674
+ mR=mR,
2675
+ limits_type=mr_spec.limits_type,
2676
+ round_to=spec.round_to,
2677
+ )
2678
+ else:
2679
+ center_val = mR
2680
+ lims = calculate_limits(
2681
+ mean=0,
2682
+ sd=0,
2683
+ N=0,
2684
+ mR=mR,
2685
+ limits_type=mr_spec.limits_type,
2686
+ round_to=spec.round_to,
2687
+ )
2688
+
2689
+ out['center'] = center_val
2690
+ out['mR'] = mR
2691
+ out['lpl'] = lims['lpl']
2692
+ out['upl'] = lims['upl']
2693
+
2694
+ # Detect beyond limits
2695
+ out = self._add_beyond_limits_flag(out, value_col=plot_col)
2696
+
2697
+ # Format output
2698
+ chart_out = self._build_output_columns(
2699
+ df=out,
2700
+ value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
2701
+ )
2702
+
2703
+ chart_statistics = {
2704
+ 'N': _mr_family_subgroup_n(mr_spec.chart_type),
2705
+ 'center': round(center_val, spec.round_to),
2706
+ 'lpl': round(lims['lpl'], spec.round_to),
2707
+ 'upl': round(lims['upl'], spec.round_to),
2708
+ }
2709
+
2710
+ # Adjust lane boundaries for chart type (no-op when offset == 0)
2711
+ if mr_spec.lane_boundary_offset != 0:
2712
+ lane_boundaries = _shift_lane_boundaries(lane_boundaries, mr_spec.lane_boundary_offset)
2713
+
2714
+ result.update(_build_ungrouped_mr_payload(
2715
+ mr_spec=mr_spec,
2716
+ chart_out=chart_out,
2717
+ chart_statistics=chart_statistics,
2718
+ plot_col=plot_col,
2719
+ lane_boundaries=lane_boundaries,
2720
+ ))
2721
+
2722
+ # Optionally include intermediates for companion calculation
2723
+ if _return_intermediates:
2724
+ result['_intermediates'] = {
2725
+ 'out': out,
2726
+ 'mR': mR,
2727
+ 'lane_boundaries': lane_boundaries,
2728
+ 'is_stratified': False,
2729
+ }
2730
+
2731
+ return result
2732
+
2733
+ def _calculate_xmr(
2734
+ self,
2735
+ value_col: str | None = None,
2736
+ _return_intermediates: bool = False,
2737
+ ) -> dict:
2738
+ """
2739
+ Calculate X (Individual) chart statistics.
2740
+
2741
+ Delegates to _calculate_mr_chart with _XMR_SPEC.
2742
+
2743
+ Parameters
2744
+ ----------
2745
+ value_col : str, optional
2746
+ Column to use for calculations. Defaults to spec.response_var.
2747
+ _return_intermediates : bool, optional
2748
+ If True, includes '_intermediates' key for companion-mode reuse.
2749
+
2750
+ Returns
2751
+ -------
2752
+ dict
2753
+ Chart data: {'X': {'data': df, 'statistics': dict, 'metadata': dict}}
2754
+ """
2755
+ return self._calculate_mr_chart(
2756
+ _XMR_SPEC,
2757
+ value_col=value_col,
2758
+ _return_intermediates=_return_intermediates,
2759
+ )
2760
+
2761
+ def _calculate_r(
2762
+ self,
2763
+ value_col: str | None = None,
2764
+ _precomputed: dict | None = None,
2765
+ ) -> dict:
2766
+ """
2767
+ Calculate mR (Moving Range) chart statistics.
2768
+
2769
+ Delegates to _calculate_mr_chart with _R_SPEC.
2770
+
2771
+ Parameters
2772
+ ----------
2773
+ value_col : str, optional
2774
+ Column to use for calculations. Defaults to spec.response_var.
2775
+ _precomputed : dict, optional
2776
+ Pre-computed intermediates from _calculate_xmr() for companion mode.
2777
+
2778
+ Returns
2779
+ -------
2780
+ dict
2781
+ Chart data: {'mR': {'data': df, 'statistics': dict, 'metadata': dict}}
2782
+ """
2783
+ return self._calculate_mr_chart(
2784
+ _R_SPEC,
2785
+ value_col=value_col,
2786
+ _precomputed=_precomputed,
2787
+ )
2788
+
2789
+ def _calculate_xmr_r(self, value_col: str | None = None) -> dict[str, ChartPayload]:
2790
+ """
2791
+ Calculate X and mR charts together (Bishop methodology).
2792
+
2793
+ This method ensures that when companion=True, both charts are calculated
2794
+ efficiently using shared intermediate values. The mR chart uses the
2795
+ same data computed for X, avoiding redundant calculation.
2796
+
2797
+ Parameters
2798
+ ----------
2799
+ value_col : str, optional
2800
+ Column to use for calculations. Defaults to spec.response_var.
2801
+
2802
+ Returns
2803
+ -------
2804
+ dict
2805
+ Combined chart data: {'X': {...}, 'mR': {...}}
2806
+ """
2807
+ # Calculate X with intermediates for mR reuse
2808
+ xmr_result = self._calculate_xmr(value_col, _return_intermediates=True)
2809
+
2810
+ # Extract intermediates and remove from result
2811
+ intermediates = xmr_result.pop('_intermediates')
2812
+
2813
+ # Calculate mR using precomputed values
2814
+ r_result = self._calculate_r(value_col, _precomputed=intermediates)
2815
+
2816
+ # Combine results
2817
+ return {**xmr_result, **r_result}
2818
+
2819
+ def _calculate_histogram(self, value_col: str | None = None) -> dict[str, ChartPayload]:
2820
+ """
2821
+ Calculate histogram data with mean/std statistics.
2822
+
2823
+ Parameters
2824
+ ----------
2825
+ value_col : str, optional
2826
+ Column to use for histogram. Defaults to spec.response_var.
2827
+ Pass a residual column (R2, R5, etc.) for residual histograms.
2828
+
2829
+ Returns
2830
+ -------
2831
+ dict
2832
+ Histogram data:
2833
+ - For unstratified: {'Histogram': {'data': df, 'statistics': {...}, 'metadata': {...}}}
2834
+ - For stratified: {'Histogram': {'data': df, 'statistics': {stratum: {...}}, 'strata': [...]}}
2835
+
2836
+ Notes
2837
+ -----
2838
+ - by=[] → single histogram of all observations (default for Histogram chart)
2839
+ - by=['factor1'] → one histogram per factor1 level
2840
+ - Multi-column `by` uses tuple keys to avoid collision from string joining
2841
+ """
2842
+ spec = self.spec
2843
+ if value_col is None:
2844
+ value_col = self.request.value_col if self.request.value_col else spec.response_var
2845
+
2846
+ data = self._source_frame().copy()
2847
+ values = data[value_col].dropna()
2848
+
2849
+ # Calculate global statistics
2850
+ n = len(values)
2851
+ mean = values.mean() if n > 0 else float('nan')
2852
+ std = values.std() if n >= 2 else float('nan')
2853
+
2854
+ # Handle stratification via `by` parameter
2855
+ by = list(self.request.by) if self.request.by is not None else []
2856
+
2857
+ if len(by) > 0:
2858
+ # Stratified histogram - one per stratum
2859
+ # Use pandas groupby with list of columns (no collision risk)
2860
+ grouped = data.groupby(by, observed=True)
2861
+
2862
+ def _py(v):
2863
+ """Normalize numpy/pandas scalars to python natives."""
2864
+ return v.item() if hasattr(v, 'item') else v
2865
+
2866
+ # Build strata list - strings for all cases via encode_rsg
2867
+ if len(by) == 1:
2868
+ strata = data[by[0]].unique().tolist()
2869
+ else:
2870
+ strata = [encode_rsg(tuple(_py(v) for v in k)) for k in grouped.groups]
2871
+
2872
+ # Calculate per-stratum statistics
2873
+ per_stratum_stats = {}
2874
+ for stratum, group_df in grouped:
2875
+ # Unwrap single-element tuple for single-column groupby
2876
+ # groupby(['col']) returns ('value',) tuples, but strata uses scalar values
2877
+ key = stratum[0] if len(by) == 1 else encode_rsg(tuple(_py(v) for v in stratum))
2878
+ stratum_vals = group_df[value_col].dropna()
2879
+ stratum_n = len(stratum_vals)
2880
+ stratum_mean = stratum_vals.mean() if stratum_n > 0 else float('nan')
2881
+ stratum_std = stratum_vals.std() if stratum_n >= 2 else float('nan')
2882
+ _mean_rounded = round(stratum_mean, spec.round_to) if pd.notna(stratum_mean) else None
2883
+ _std_rounded = round(stratum_std, spec.round_to) if pd.notna(stratum_std) else None
2884
+ # Unified stats shape ({N, center, lpl, upl}) plus histogram
2885
+ # extras (mean alias, std). Histogram has no control limits.
2886
+ per_stratum_stats[key] = {
2887
+ 'N': stratum_n,
2888
+ 'center': _mean_rounded,
2889
+ 'lpl': None,
2890
+ 'upl': None,
2891
+ 'mean': _mean_rounded,
2892
+ 'std': _std_rounded,
2893
+ 'n': stratum_n,
2894
+ }
2895
+
2896
+ # Keep value column and by columns for plotting
2897
+ output_cols = [value_col] + list(by)
2898
+ output_data = data[output_cols].copy()
2899
+
2900
+ # Add rsg column so focus() can filter consistently with other chart types
2901
+
2902
+ if len(by) == 1:
2903
+ output_data['rsg'] = encode_rsg_series(output_data, [by[0]])
2904
+ else:
2905
+ output_data['rsg'] = encode_rsg_series(output_data, list(by))
2906
+
2907
+ return {
2908
+ 'Histogram': {
2909
+ 'data': output_data,
2910
+ 'statistics': per_stratum_stats,
2911
+ 'metadata': {
2912
+ 'chart_type': 'Histogram',
2913
+ 'value_col': value_col,
2914
+ 'bins': self.request.bins,
2915
+ 'stratified': True,
2916
+ 'stratify_col': by[0] if len(by) == 1 else '_stratify_key',
2917
+ 'stratify_by': list(by),
2918
+ },
2919
+ 'strata': strata,
2920
+ }
2921
+ }
2922
+
2923
+ # Unstratified (by=[]) - full distribution
2924
+ output_data = data[[value_col]].copy()
2925
+ _mean_rounded = round(mean, spec.round_to) if pd.notna(mean) else None
2926
+ _std_rounded = round(std, spec.round_to) if n >= 2 and pd.notna(std) else None
2927
+
2928
+ return {
2929
+ 'Histogram': {
2930
+ 'data': output_data,
2931
+ # Unified stats shape ({N, center, lpl, upl}) plus histogram
2932
+ # extras (mean alias, std). Histogram has no control limits.
2933
+ 'statistics': {
2934
+ 'N': n,
2935
+ 'center': _mean_rounded,
2936
+ 'lpl': None,
2937
+ 'upl': None,
2938
+ 'mean': _mean_rounded,
2939
+ 'std': _std_rounded,
2940
+ 'n': n,
2941
+ },
2942
+ 'metadata': {'chart_type': 'Histogram', 'value_col': value_col, 'bins': self.request.bins},
2943
+ }
2944
+ }