processbehavior 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- processbehavior/__init__.py +133 -0
- processbehavior/analysis.py +2944 -0
- processbehavior/analysis_dataset.py +361 -0
- processbehavior/analysis_result.py +1276 -0
- processbehavior/calibration.py +70 -0
- processbehavior/capability.py +720 -0
- processbehavior/data_preparation.py +792 -0
- processbehavior/datasets/__init__.py +37 -0
- processbehavior/datasets/data/coffee_shop_demo_long.csv +385 -0
- processbehavior/datasets/loaders.py +66 -0
- processbehavior/datasets/synthetic.py +1711 -0
- processbehavior/derivations.py +602 -0
- processbehavior/effects_calculator.py +603 -0
- processbehavior/exceptions.py +216 -0
- processbehavior/exporters/__init__.py +5 -0
- processbehavior/exporters/excel_exporter.py +708 -0
- processbehavior/focused_result.py +329 -0
- processbehavior/formulation_spec.py +144 -0
- processbehavior/loss_function.py +517 -0
- processbehavior/maximum_information.py +285 -0
- processbehavior/plotting/__init__.py +46 -0
- processbehavior/plotting/_math.py +121 -0
- processbehavior/plotting/capability_chart.py +858 -0
- processbehavior/plotting/contracts.py +176 -0
- processbehavior/plotting/control_chart.py +206 -0
- processbehavior/plotting/effects_charts.py +640 -0
- processbehavior/plotting/lane_boundaries.py +78 -0
- processbehavior/plotting/limits.py +157 -0
- processbehavior/plotting/maximum_information_chart.py +395 -0
- processbehavior/plotting/plotter.py +1469 -0
- processbehavior/plotting/renderers.py +660 -0
- processbehavior/plotting/report.py +211 -0
- processbehavior/plotting/residuals.py +282 -0
- processbehavior/plotting/run_rules_viz.py +150 -0
- processbehavior/plotting/stats_box.py +183 -0
- processbehavior/plotting/themes.py +544 -0
- processbehavior/plotting/x_axis_layout.py +474 -0
- processbehavior/plotting/zones.py +127 -0
- processbehavior/process_behavior.py +1290 -0
- processbehavior/py.typed +0 -0
- processbehavior/residual_calculator.py +572 -0
- processbehavior/result_signals.py +149 -0
- processbehavior/result_tabular.py +296 -0
- processbehavior/sds_detector.py +1432 -0
- processbehavior/signals/__init__.py +14 -0
- processbehavior/signals/config.py +435 -0
- processbehavior/signals/detector.py +328 -0
- processbehavior/signals/detectors.py +368 -0
- processbehavior/signals/result.py +232 -0
- processbehavior/spc_constants.py +692 -0
- processbehavior/study.py +2733 -0
- processbehavior/types.py +154 -0
- processbehavior-0.2.0.dist-info/METADATA +279 -0
- processbehavior-0.2.0.dist-info/RECORD +56 -0
- processbehavior-0.2.0.dist-info/WHEEL +4 -0
- processbehavior-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,2944 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Analysis - Chart calculation strategies for process behavior analysis.
|
|
3
|
+
|
|
4
|
+
This module provides the Analysis class which executes chart calculations using
|
|
5
|
+
the strategy pattern. It supports:
|
|
6
|
+
- Xbar and S charts (subgroup mean and variation)
|
|
7
|
+
- X charts (individual values)
|
|
8
|
+
- mR charts (moving range)
|
|
9
|
+
|
|
10
|
+
The Analysis class coordinates between:
|
|
11
|
+
- FormulationSpec (structural configuration)
|
|
12
|
+
- ChartRequest (per-execute chart parameters)
|
|
13
|
+
- AnalysisDataSet (data preparation and VAS calculations)
|
|
14
|
+
- Chart calculation strategies
|
|
15
|
+
- AnalysisResult (unified result container)
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
spec = FormulationSpec(response_var='Height', rsg_vars=('Operator', 'Machine'), time_var='Time')
|
|
19
|
+
request = ChartRequest(chart='Xbar')
|
|
20
|
+
analysis = Analysis(spec, request, analysis_dataset=ads)
|
|
21
|
+
result = analysis.calculate()
|
|
22
|
+
|
|
23
|
+
# Access charts
|
|
24
|
+
xbar = result.get_chart('Xbar')
|
|
25
|
+
s = result.get_chart('S')
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import logging
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from functools import reduce
|
|
33
|
+
|
|
34
|
+
import numpy as np
|
|
35
|
+
import pandas as pd
|
|
36
|
+
|
|
37
|
+
from .analysis_dataset import AnalysisDataSet
|
|
38
|
+
from .analysis_result import AnalysisResult
|
|
39
|
+
from .data_preparation import encode_rsg, encode_rsg_series
|
|
40
|
+
from .exceptions import CalibrationNotSupportedError, ChartNotAvailableError, ValidationError
|
|
41
|
+
from .formulation_spec import ChartRequest, FormulationSpec
|
|
42
|
+
from .residual_calculator import calculate_r6_residuals, resolve_r6_groupby
|
|
43
|
+
from .spc_constants import (
|
|
44
|
+
REQUEST_RESIDUALS,
|
|
45
|
+
calculate_limits,
|
|
46
|
+
calculate_limits_vectorized,
|
|
47
|
+
calibrated_limits,
|
|
48
|
+
)
|
|
49
|
+
from .types import ChartPayload
|
|
50
|
+
|
|
51
|
+
# Configure module logger
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ============================================================================
|
|
56
|
+
# MR-Family Chart Specification
|
|
57
|
+
# ============================================================================
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class _MRChartSpec:
|
|
62
|
+
"""Captures what makes an MR-family chart different from its sibling.
|
|
63
|
+
|
|
64
|
+
X and mR charts share >85% of their calculation pipeline. The differences
|
|
65
|
+
are behavioral, not parametric — this dataclass encodes those differences
|
|
66
|
+
so a single shared method can serve both chart types.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
chart_type: str # 'X' or 'mR'
|
|
70
|
+
limits_type: str # "XmR" or "R" — passed to calculate_limits()
|
|
71
|
+
plot_col: str # Which column to plot: 'raw' (use value_col) or 'mr'
|
|
72
|
+
center_source: str # What the center line represents: 'mean' or 'mR'
|
|
73
|
+
drops_first_mr: bool # False for X, True for mR
|
|
74
|
+
lane_boundary_offset: int # 0 for X, -1 for mR
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _mr_family_subgroup_n(chart_type: str) -> int:
|
|
78
|
+
"""Subgroup size to report in an MR-family chart's statistics dict.
|
|
79
|
+
|
|
80
|
+
An X chart plots one observation per point; an mR chart plots the range of
|
|
81
|
+
a two-observation moving window. Reporting these keeps ``statistics['N']``
|
|
82
|
+
meaning the same thing — subgroup size — on every chart family, which is
|
|
83
|
+
what ``get_statistics`` documents. The X/mR paths used to omit the key
|
|
84
|
+
entirely, so code written against the documented four-key contract raised
|
|
85
|
+
``KeyError`` on exactly the charts where it was promised.
|
|
86
|
+
"""
|
|
87
|
+
return 2 if chart_type == 'mR' else 1
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
_XMR_SPEC = _MRChartSpec(
|
|
91
|
+
chart_type='X',
|
|
92
|
+
limits_type='XmR',
|
|
93
|
+
plot_col='raw',
|
|
94
|
+
center_source='mean',
|
|
95
|
+
drops_first_mr=False,
|
|
96
|
+
lane_boundary_offset=0,
|
|
97
|
+
)
|
|
98
|
+
_R_SPEC = _MRChartSpec(
|
|
99
|
+
chart_type='mR',
|
|
100
|
+
limits_type='R',
|
|
101
|
+
plot_col='mr',
|
|
102
|
+
center_source='mR',
|
|
103
|
+
drops_first_mr=True,
|
|
104
|
+
lane_boundary_offset=-1,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _join_limits(grouped: pd.DataFrame, lims_result, rsuffix: str = '') -> pd.DataFrame:
|
|
109
|
+
"""Join lpl/upl from a DataFrame.apply(calculate_limits) result.
|
|
110
|
+
|
|
111
|
+
``DataFrame.apply()`` with ``calculate_limits`` returns either:
|
|
112
|
+
- A ``pd.DataFrame`` (when all Series share the same index), or
|
|
113
|
+
- A ``pd.Series`` of ``pd.Series`` objects.
|
|
114
|
+
|
|
115
|
+
This helper normalizes both cases and joins ``lpl``/``upl`` onto *grouped*.
|
|
116
|
+
"""
|
|
117
|
+
if isinstance(lims_result, pd.DataFrame):
|
|
118
|
+
return grouped.join(lims_result[['lpl', 'upl']], rsuffix=rsuffix)
|
|
119
|
+
|
|
120
|
+
# Series of Series — extract lpl/upl manually
|
|
121
|
+
lims_df = pd.DataFrame(
|
|
122
|
+
[(row['lpl'], row['upl']) for row in lims_result],
|
|
123
|
+
index=grouped.index,
|
|
124
|
+
columns=['lpl', 'upl'],
|
|
125
|
+
)
|
|
126
|
+
return grouped.join(lims_df, rsuffix=rsuffix)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
# MR-chart payload assembly helpers
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
#
|
|
133
|
+
# The four `_calculate_mr_chart_*` paths (precomputed/independent ×
|
|
134
|
+
# stratified/ungrouped, phased and global) all end with near-identical
|
|
135
|
+
# code that adjusts lane boundaries for the dropped-first-mR row,
|
|
136
|
+
# filters insufficient strata, and assembles the output dict.
|
|
137
|
+
# These helpers consolidate that tail so adding a new metadata key, or
|
|
138
|
+
# fixing a strata-filter bug, happens in one place.
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _shift_lane_boundaries(lane_boundaries, offset: int):
|
|
142
|
+
"""Shift each boundary's `position` by `offset`, drop those that go negative.
|
|
143
|
+
|
|
144
|
+
Accepts the flat (`list[dict]`) shape or the stratified
|
|
145
|
+
(`dict[stratum, list[dict]]`) shape. Returns the same shape; returns
|
|
146
|
+
None when the input is empty or every position was dropped.
|
|
147
|
+
"""
|
|
148
|
+
if not lane_boundaries:
|
|
149
|
+
return None
|
|
150
|
+
if isinstance(lane_boundaries, dict):
|
|
151
|
+
adjusted: dict[str, list[dict]] = {}
|
|
152
|
+
for stratum, boundaries in lane_boundaries.items():
|
|
153
|
+
shifted = [
|
|
154
|
+
{**b, 'position': b['position'] + offset}
|
|
155
|
+
for b in boundaries
|
|
156
|
+
if b['position'] + offset >= 0
|
|
157
|
+
]
|
|
158
|
+
if shifted:
|
|
159
|
+
adjusted[stratum] = shifted
|
|
160
|
+
return adjusted or None
|
|
161
|
+
shifted = [
|
|
162
|
+
{**b, 'position': b['position'] + offset}
|
|
163
|
+
for b in lane_boundaries
|
|
164
|
+
if b['position'] + offset >= 0
|
|
165
|
+
]
|
|
166
|
+
return shifted or None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _split_strata_by_sufficiency(
|
|
170
|
+
out: pd.DataFrame, stratify_col: str, strata: list, min_obs: int = 2
|
|
171
|
+
) -> tuple[list, list]:
|
|
172
|
+
"""Partition `strata` into (insufficient, published) by post-mR row count.
|
|
173
|
+
|
|
174
|
+
The focus()/strata contract requires every published stratum to be
|
|
175
|
+
focusable; insufficient strata have empty (or near-empty) data after
|
|
176
|
+
the first-mR drop and would fail downstream consumers.
|
|
177
|
+
|
|
178
|
+
Returns (insufficient_strata, published_strata).
|
|
179
|
+
|
|
180
|
+
One counting pass, not one table scan per stratum. The obvious form —
|
|
181
|
+
|
|
182
|
+
[s for s in strata if len(out[out[stratify_col] == s]) < min_obs]
|
|
183
|
+
|
|
184
|
+
compares the whole column and materialises a filtered frame for *every* stratum, which
|
|
185
|
+
is O(strata x rows): 2.4s at 200K rows / 632 strata against 0.006s here, and it was 89%
|
|
186
|
+
of a stratified execute. The `published` membership test uses a set for the same
|
|
187
|
+
reason — against a list it is O(strata^2), invisible at 632 strata and quadratic beyond.
|
|
188
|
+
|
|
189
|
+
NaN strata count as 0 either way: value_counts() drops NaN, and `col == NaN` is always
|
|
190
|
+
False.
|
|
191
|
+
"""
|
|
192
|
+
counts = out[stratify_col].value_counts()
|
|
193
|
+
insufficient = [s for s in strata if counts.get(s, 0) < min_obs]
|
|
194
|
+
insufficient_set = set(insufficient)
|
|
195
|
+
published = [s for s in strata if s not in insufficient_set]
|
|
196
|
+
return insufficient, published
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _no_replication_alternative(chart: str) -> str:
|
|
200
|
+
"""What to reach for instead, when a subgroup chart has nothing to subgroup."""
|
|
201
|
+
return (
|
|
202
|
+
"Use chart='X' for individual values, or chart='Xbar' with value='R6' for effects analysis."
|
|
203
|
+
if chart == 'Xbar'
|
|
204
|
+
else "Use chart='X' for individual values."
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _raise_no_replicated_subgroups(chart: str, sds) -> None:
|
|
209
|
+
"""Raise for an *ungrouped* Xbar/S chart with nothing to subgroup.
|
|
210
|
+
|
|
211
|
+
Sibling of :func:`_raise_no_replication`, which serves the stratified paths and
|
|
212
|
+
adds the per-stratum detail. Both are the enforcement half of the rule
|
|
213
|
+
:meth:`Study._response_pair_problem` reports in advance, so ``why_not()`` and
|
|
214
|
+
``execute()`` can no longer disagree about whether Xbar can chart the response.
|
|
215
|
+
Change one, change all three.
|
|
216
|
+
"""
|
|
217
|
+
raise ValidationError(
|
|
218
|
+
f'No subgroups with n > 1 found — {chart} chart requires replicated observations.\n'
|
|
219
|
+
f'This data has Analytical Design State {sds}.\n'
|
|
220
|
+
f'{_no_replication_alternative(chart)}'
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _raise_no_replication(chart: str, sds, insufficient: list, total: int) -> None:
|
|
225
|
+
"""Raise the self-diagnostic error for a subgroup chart with nothing to subgroup.
|
|
226
|
+
|
|
227
|
+
Xbar/S both need n >= 2 within a subgroup; with every subgroup at n = 1 there is no
|
|
228
|
+
within-subgroup dispersion to chart. Shared by the stratified paths so the stratified
|
|
229
|
+
message matches the ungrouped one — the stratified S path used to fall through to
|
|
230
|
+
``pd.concat([])`` and surface a raw pandas "No objects to concatenate" instead.
|
|
231
|
+
"""
|
|
232
|
+
detail = {2: ' (no replication)', 3: ' (partial replication)'}.get(sds, '')
|
|
233
|
+
alternative = _no_replication_alternative(chart)
|
|
234
|
+
raise ValidationError(
|
|
235
|
+
f'No subgroups with n > 1 found — {chart} chart requires replicated observations.\n'
|
|
236
|
+
f'This data has Analytical Design State {sds}{detail}.\n'
|
|
237
|
+
f'All {total} strata were unusable: {insufficient}.\n'
|
|
238
|
+
f'{alternative}'
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _build_stratified_mr_payload(
|
|
243
|
+
*,
|
|
244
|
+
mr_spec: _MRChartSpec,
|
|
245
|
+
chart_out: pd.DataFrame,
|
|
246
|
+
chart_statistics: dict,
|
|
247
|
+
plot_col: str,
|
|
248
|
+
stratify_col: str,
|
|
249
|
+
stratify_by,
|
|
250
|
+
strata: list,
|
|
251
|
+
lane_boundaries,
|
|
252
|
+
phased: bool = False,
|
|
253
|
+
insufficient_strata: list | None = None,
|
|
254
|
+
single_point_phases: int | None = None,
|
|
255
|
+
) -> dict:
|
|
256
|
+
"""Build the {chart_type: {...}} payload for a stratified MR-family chart.
|
|
257
|
+
|
|
258
|
+
For phased layouts `strata` is published as-is. For non-phased
|
|
259
|
+
layouts, callers pre-compute `insufficient_strata` via
|
|
260
|
+
`_split_strata_by_sufficiency` and pass them in; published strata
|
|
261
|
+
are derived here.
|
|
262
|
+
"""
|
|
263
|
+
metadata: dict = {
|
|
264
|
+
'chart_type': mr_spec.chart_type,
|
|
265
|
+
'value_col': plot_col,
|
|
266
|
+
'center_col': 'center',
|
|
267
|
+
'stratified': True,
|
|
268
|
+
'lane_boundaries': lane_boundaries if lane_boundaries else None,
|
|
269
|
+
'stratify_col': stratify_col,
|
|
270
|
+
'stratify_by': list(stratify_by),
|
|
271
|
+
}
|
|
272
|
+
if phased:
|
|
273
|
+
metadata['phased'] = True
|
|
274
|
+
metadata['run_rules_applicable'] = False
|
|
275
|
+
if single_point_phases is not None:
|
|
276
|
+
metadata['single_point_phases'] = single_point_phases
|
|
277
|
+
published_strata = list(strata)
|
|
278
|
+
else:
|
|
279
|
+
metadata['insufficient_strata'] = insufficient_strata or None
|
|
280
|
+
published_strata = [s for s in strata if s not in (insufficient_strata or [])]
|
|
281
|
+
return {
|
|
282
|
+
mr_spec.chart_type: {
|
|
283
|
+
'data': chart_out,
|
|
284
|
+
'statistics': chart_statistics,
|
|
285
|
+
'metadata': metadata,
|
|
286
|
+
'strata': published_strata,
|
|
287
|
+
},
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _build_ungrouped_mr_payload(
|
|
292
|
+
*,
|
|
293
|
+
mr_spec: _MRChartSpec,
|
|
294
|
+
chart_out: pd.DataFrame,
|
|
295
|
+
chart_statistics: dict,
|
|
296
|
+
plot_col: str,
|
|
297
|
+
lane_boundaries,
|
|
298
|
+
phased: bool = False,
|
|
299
|
+
single_point_phases: int | None = None,
|
|
300
|
+
) -> dict:
|
|
301
|
+
"""Build the {chart_type: {...}} payload for an ungrouped MR-family chart."""
|
|
302
|
+
metadata: dict = {
|
|
303
|
+
'chart_type': mr_spec.chart_type,
|
|
304
|
+
'value_col': plot_col,
|
|
305
|
+
'center_col': 'center',
|
|
306
|
+
'lane_boundaries': lane_boundaries,
|
|
307
|
+
}
|
|
308
|
+
if phased:
|
|
309
|
+
metadata['phased'] = True
|
|
310
|
+
metadata['run_rules_applicable'] = False
|
|
311
|
+
if single_point_phases is not None:
|
|
312
|
+
metadata['single_point_phases'] = single_point_phases
|
|
313
|
+
return {
|
|
314
|
+
mr_spec.chart_type: {
|
|
315
|
+
'data': chart_out,
|
|
316
|
+
'statistics': chart_statistics,
|
|
317
|
+
'metadata': metadata,
|
|
318
|
+
},
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# Chart dispatch tables — added by analysis type and companion mode.
|
|
323
|
+
# Values are method names on Analysis; the orchestrator calls
|
|
324
|
+
# `getattr(self, name)(...)` so this stays a single source of truth for
|
|
325
|
+
# "which method handles this chart type?". Adding a new chart type
|
|
326
|
+
# becomes adding two rows.
|
|
327
|
+
_COMPANION_STRATEGY_MAP: dict[str, str] = {
|
|
328
|
+
'Xbar': '_calculate_xbar_s',
|
|
329
|
+
'S': '_calculate_xbar_s',
|
|
330
|
+
'X': '_calculate_xmr_r',
|
|
331
|
+
'mR': '_calculate_xmr_r',
|
|
332
|
+
'Histogram': '_calculate_histogram', # no companion exists; degrade to standalone
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
_SOLO_STRATEGY_MAP: dict[str, str] = {
|
|
336
|
+
'Xbar': '_calculate_xbar',
|
|
337
|
+
'S': '_calculate_s',
|
|
338
|
+
'X': '_calculate_xmr',
|
|
339
|
+
'mR': '_calculate_r',
|
|
340
|
+
'Histogram': '_calculate_histogram',
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
# Residual charts call the companion-style methods (Xbar_S / Xmr_R)
|
|
344
|
+
# when companion=True, the solo methods otherwise. Both tables mirror their
|
|
345
|
+
# response-chart counterparts entry for entry — residual context uses the same
|
|
346
|
+
# five base entry points.
|
|
347
|
+
_RESIDUAL_COMPANION_STRATEGY_MAP: dict[str, str] = {
|
|
348
|
+
'Xbar': '_calculate_xbar_s',
|
|
349
|
+
'S': '_calculate_xbar_s',
|
|
350
|
+
'X': '_calculate_xmr_r',
|
|
351
|
+
'mR': '_calculate_xmr_r',
|
|
352
|
+
'Histogram': '_calculate_histogram',
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
_RESIDUAL_SOLO_STRATEGY_MAP: dict[str, str] = {
|
|
356
|
+
'Xbar': '_calculate_xbar',
|
|
357
|
+
'S': '_calculate_s',
|
|
358
|
+
'X': '_calculate_xmr',
|
|
359
|
+
# 'mR' was missing here while _SOLO_STRATEGY_MAP had it, so a standalone
|
|
360
|
+
# moving-range chart of residuals was unreachable even though the companion
|
|
361
|
+
# path computed exactly that. See tests/test_solo_mr_residuals.py.
|
|
362
|
+
'mR': '_calculate_r',
|
|
363
|
+
'Histogram': '_calculate_histogram',
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
class Analysis:
|
|
368
|
+
"""
|
|
369
|
+
Unified analysis class handling all chart types via strategy pattern.
|
|
370
|
+
|
|
371
|
+
This class replaces the AbstractFactory pattern with a simpler, more maintainable
|
|
372
|
+
approach. All analysis types (Xbar, S, X, mR) are handled through internal
|
|
373
|
+
strategy methods.
|
|
374
|
+
|
|
375
|
+
Usage:
|
|
376
|
+
# Standard usage (calculates AnalysisDataSet from scratch)
|
|
377
|
+
analysis = Analysis(df, specification)
|
|
378
|
+
result = analysis.calculate()
|
|
379
|
+
|
|
380
|
+
# With pre-calculated AnalysisDataSet (for Study.execute())
|
|
381
|
+
analysis = Analysis(df, specification, analysis_dataset=ads)
|
|
382
|
+
result = analysis.calculate() # Reuses pre-calculated data
|
|
383
|
+
"""
|
|
384
|
+
|
|
385
|
+
def __init__(
|
|
386
|
+
self,
|
|
387
|
+
spec: FormulationSpec,
|
|
388
|
+
request: ChartRequest,
|
|
389
|
+
analysis_dataset: AnalysisDataSet | None = None,
|
|
390
|
+
sds: int | None = None,
|
|
391
|
+
df: pd.DataFrame | None = None,
|
|
392
|
+
):
|
|
393
|
+
"""
|
|
394
|
+
Initialize analysis with spec and chart request.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
spec: Structural configuration (from formulate())
|
|
398
|
+
request: Chart-specific request (from execute())
|
|
399
|
+
analysis_dataset: Optional pre-calculated AnalysisDataSet.
|
|
400
|
+
If provided, skips expensive residual calculation.
|
|
401
|
+
Used by Study.execute() to reuse formulate() calculations.
|
|
402
|
+
sds: Sampling Design State (0-6). Required if analysis_dataset is not
|
|
403
|
+
provided. SDS should be detected at the entry point (ProcessBehavior)
|
|
404
|
+
and passed through the system.
|
|
405
|
+
df: Raw DataFrame. Required if analysis_dataset is not provided.
|
|
406
|
+
|
|
407
|
+
Raises:
|
|
408
|
+
ValueError: If neither analysis_dataset nor sds is provided.
|
|
409
|
+
"""
|
|
410
|
+
self.spec = spec
|
|
411
|
+
self.request = request
|
|
412
|
+
self.analysis_type = request.chart
|
|
413
|
+
|
|
414
|
+
# Calibration (standards-given limits). center_zero applies to a plain
|
|
415
|
+
# residual chart (residual set, not recentered): its mean is meaningless,
|
|
416
|
+
# so the center is pinned at 0. Raw response and recentered residuals use
|
|
417
|
+
# the calibration mean.
|
|
418
|
+
self.calibration = request.calibration
|
|
419
|
+
self._calibration_center_zero = bool(request.residual) and not request.recentered
|
|
420
|
+
|
|
421
|
+
# Per-request frame for request residuals (R6/RCR6): built in calculate(),
|
|
422
|
+
# None for every other request. The shared study-level frame is never
|
|
423
|
+
# mutated — strategies read through _source_frame().
|
|
424
|
+
self._request_frame: pd.DataFrame | None = None
|
|
425
|
+
|
|
426
|
+
# Use pre-calculated AnalysisDataSet if provided, otherwise calculate
|
|
427
|
+
if analysis_dataset is not None:
|
|
428
|
+
self.ads = analysis_dataset
|
|
429
|
+
else:
|
|
430
|
+
if sds is None:
|
|
431
|
+
raise ValidationError(
|
|
432
|
+
'sds is required when analysis_dataset is not provided. '
|
|
433
|
+
'SDS should be detected at the entry point (ProcessBehavior) '
|
|
434
|
+
'and passed to Analysis.'
|
|
435
|
+
)
|
|
436
|
+
if df is None:
|
|
437
|
+
raise ValidationError('df is required when analysis_dataset is not provided.')
|
|
438
|
+
self.ads = AnalysisDataSet(df, spec, observed_sds=sds)
|
|
439
|
+
|
|
440
|
+
def calculate(self) -> AnalysisResult:
|
|
441
|
+
"""
|
|
442
|
+
Execute the appropriate analysis strategy and return comprehensive results.
|
|
443
|
+
|
|
444
|
+
Returns
|
|
445
|
+
-------
|
|
446
|
+
AnalysisResult
|
|
447
|
+
Unified result object containing:
|
|
448
|
+
- charts: Chart data and statistics
|
|
449
|
+
- residuals: VAS residuals (R1-R5) if calculated
|
|
450
|
+
- effects: Main effects if calculated
|
|
451
|
+
- interactions: Interaction effects if calculated
|
|
452
|
+
- summary: Comprehensive metadata
|
|
453
|
+
- dataset: Full analysis dataset
|
|
454
|
+
|
|
455
|
+
Raises
|
|
456
|
+
------
|
|
457
|
+
ValueError
|
|
458
|
+
If analysis_type is not supported
|
|
459
|
+
|
|
460
|
+
Examples
|
|
461
|
+
--------
|
|
462
|
+
>>> result = analysis.calculate()
|
|
463
|
+
>>> xbar = result.get_chart('Xbar')
|
|
464
|
+
>>> if result.has_residuals:
|
|
465
|
+
... residuals = result.residuals
|
|
466
|
+
"""
|
|
467
|
+
# Check if this is a residual chart request
|
|
468
|
+
residual = self.request.residual
|
|
469
|
+
if residual:
|
|
470
|
+
# Residual chart analysis - inline logic (was _calculate_residual_chart)
|
|
471
|
+
chart_type = self.request.residual_chart_type or 'X'
|
|
472
|
+
recentered = self.request.recentered
|
|
473
|
+
|
|
474
|
+
# Request residuals (R6/RCR6) are computed here, onto this request's own
|
|
475
|
+
# frame — the shared study-level frame is never mutated.
|
|
476
|
+
if residual in REQUEST_RESIDUALS:
|
|
477
|
+
self._request_frame = self._build_request_frame(recentered)
|
|
478
|
+
|
|
479
|
+
# Determine column name
|
|
480
|
+
col_prefix = 'RCR' if recentered else 'R'
|
|
481
|
+
residual_num = residual[1:] # Extract number from 'R2', 'R3', 'R10', etc.
|
|
482
|
+
col_name = f'{col_prefix}{residual_num}'
|
|
483
|
+
|
|
484
|
+
# Validate column exists
|
|
485
|
+
if col_name not in self._source_frame().columns:
|
|
486
|
+
available = [
|
|
487
|
+
c for c in self._source_frame().columns if c.startswith('R') or c == self.spec.response_var
|
|
488
|
+
]
|
|
489
|
+
raise ChartNotAvailableError(
|
|
490
|
+
f"Residual column '{col_name}' not found.\n"
|
|
491
|
+
f'Available columns: {available}\n'
|
|
492
|
+
f"This may indicate the analytical design state (ADS) doesn't support this residual type.",
|
|
493
|
+
chart=col_name,
|
|
494
|
+
available=available,
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
# Validate chart type
|
|
498
|
+
valid_chart_types = {'Xbar', 'S', 'X', 'mR', 'Histogram'}
|
|
499
|
+
if chart_type not in valid_chart_types:
|
|
500
|
+
raise ChartNotAvailableError(
|
|
501
|
+
f"Chart type '{chart_type}' not supported for residual charts.\n"
|
|
502
|
+
f'Valid types: {", ".join(sorted(valid_chart_types))}',
|
|
503
|
+
chart=chart_type,
|
|
504
|
+
available=sorted(valid_chart_types),
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# Question answered by each residual
|
|
508
|
+
questions = {
|
|
509
|
+
'R2': 'Is within-subgroup variation stable?',
|
|
510
|
+
'R3': 'Is there interaction between factors and time?',
|
|
511
|
+
'R4': 'Does time have a significant effect?',
|
|
512
|
+
'R5': 'Do factors have a significant effect?',
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
# Dispatch to the appropriate residual-chart calculator via registry.
|
|
516
|
+
companion = self.request.companion
|
|
517
|
+
registry = _RESIDUAL_COMPANION_STRATEGY_MAP if companion else _RESIDUAL_SOLO_STRATEGY_MAP
|
|
518
|
+
method_name = registry.get(chart_type)
|
|
519
|
+
if method_name is None:
|
|
520
|
+
raise ChartNotAvailableError(
|
|
521
|
+
f"Chart type '{chart_type}' not supported for residual charts.\n"
|
|
522
|
+
f'Valid types: {", ".join(sorted(registry))}',
|
|
523
|
+
chart=chart_type,
|
|
524
|
+
available=sorted(registry),
|
|
525
|
+
)
|
|
526
|
+
chart_data = getattr(self, method_name)(value_col=col_name)
|
|
527
|
+
chart_data = self._add_residual_metadata(chart_data, residual, recentered, questions)
|
|
528
|
+
|
|
529
|
+
else:
|
|
530
|
+
# Standard chart analysis — registry dispatch by chart type and companion mode.
|
|
531
|
+
companion = self.request.companion
|
|
532
|
+
registry = _COMPANION_STRATEGY_MAP if companion else _SOLO_STRATEGY_MAP
|
|
533
|
+
method_name = registry.get(self.analysis_type)
|
|
534
|
+
if method_name is None:
|
|
535
|
+
raise ChartNotAvailableError(
|
|
536
|
+
f'Analysis type {self.analysis_type} not supported! '
|
|
537
|
+
f'Valid types: {list(registry.keys())}',
|
|
538
|
+
chart=self.analysis_type,
|
|
539
|
+
available=list(registry.keys()),
|
|
540
|
+
)
|
|
541
|
+
chart_data = getattr(self, method_name)()
|
|
542
|
+
|
|
543
|
+
# Tag calibration provenance for plotting/inspection (no-op when None).
|
|
544
|
+
chart_data = self._apply_calibration_metadata(chart_data)
|
|
545
|
+
|
|
546
|
+
# Wrap in AnalysisResult for unified access
|
|
547
|
+
# Pass analysis_type so result.summary reports the executed chart type, not the recommended one
|
|
548
|
+
return AnalysisResult(
|
|
549
|
+
charts=chart_data,
|
|
550
|
+
analysis_dataset_obj=self.ads,
|
|
551
|
+
analysis_type=self.analysis_type,
|
|
552
|
+
request_dataset=self._request_frame,
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
# =========================================================================
|
|
556
|
+
# Helper Methods (DRY principle)
|
|
557
|
+
# =========================================================================
|
|
558
|
+
|
|
559
|
+
def _source_frame(self) -> pd.DataFrame:
|
|
560
|
+
"""The frame this request's strategies read from.
|
|
561
|
+
|
|
562
|
+
The per-request copy carrying R6/RCR6 for a request residual, else the
|
|
563
|
+
shared study-level frame. Callers must ``.copy()`` before mutating,
|
|
564
|
+
exactly as with the shared frame.
|
|
565
|
+
"""
|
|
566
|
+
return self._request_frame if self._request_frame is not None else self.ads.analysis_dataset
|
|
567
|
+
|
|
568
|
+
def _build_request_frame(self, recentered: bool) -> pd.DataFrame:
|
|
569
|
+
"""Compute this request's R6/RCR6 onto a copy of the study-level frame.
|
|
570
|
+
|
|
571
|
+
R6 = α_i + R2 with α_i = mean(R5 | this request's by= factor(s)) — see
|
|
572
|
+
residual_calculator.calculate_r6_residuals for the math and its
|
|
573
|
+
bit-identity constraints.
|
|
574
|
+
"""
|
|
575
|
+
shared = self.ads.analysis_dataset
|
|
576
|
+
missing = [c for c in ('R5', 'R2') if c not in shared.columns]
|
|
577
|
+
if missing:
|
|
578
|
+
available = [c for c in shared.columns if c.startswith('R') or c == self.spec.response_var]
|
|
579
|
+
raise ChartNotAvailableError(
|
|
580
|
+
f'R6 requires the stored residuals {missing} which are not present.\n'
|
|
581
|
+
f'Available columns: {available}\n'
|
|
582
|
+
f"This may indicate the analytical design state (ADS) doesn't support this residual type.",
|
|
583
|
+
chart='R6',
|
|
584
|
+
available=available,
|
|
585
|
+
)
|
|
586
|
+
groupby_key = resolve_r6_groupby(self.request.by, self.spec.rsg_vars_list)
|
|
587
|
+
return calculate_r6_residuals(shared, groupby_key, recentered)
|
|
588
|
+
|
|
589
|
+
def _resolve_xmr_stratification(self) -> tuple[list[str], list[str]]:
|
|
590
|
+
"""(stratify_by, collapsed_factors) for the X/mR path.
|
|
591
|
+
|
|
592
|
+
``by`` means *stratify by* here, not *aggregate by* — the opposite half of
|
|
593
|
+
the dual semantics that :meth:`_resolve_by_grouping` implements for Xbar/S.
|
|
594
|
+
|
|
595
|
+
Single source of truth: both :meth:`_calculate_xmr` and
|
|
596
|
+
:meth:`calibration_rejection_reason` call this. Inlining it in either place
|
|
597
|
+
is how the two answers drift apart.
|
|
598
|
+
"""
|
|
599
|
+
rsg_vars = self.spec.rsg_vars_list
|
|
600
|
+
by = list(self.request.by) if self.request.by is not None else None
|
|
601
|
+
|
|
602
|
+
if by is None:
|
|
603
|
+
# No `by` at all: one chart per factor combination.
|
|
604
|
+
return ([self.spec.rsg_var_name] if rsg_vars else []), []
|
|
605
|
+
if by == []:
|
|
606
|
+
# Explicit collapse: one chart, every factor folded into the sequence.
|
|
607
|
+
return [], list(rsg_vars)
|
|
608
|
+
return list(by), [v for v in rsg_vars if v not in by]
|
|
609
|
+
|
|
610
|
+
def calibration_rejection_reason(self) -> str | None:
|
|
611
|
+
"""The path description a calibration would be refused for, or None if it applies.
|
|
612
|
+
|
|
613
|
+
Calibration is applied on global and grouped (single-chart) Xbar/S/X/mR paths.
|
|
614
|
+
Stratified paths and phased X/mR are a documented follow-up.
|
|
615
|
+
|
|
616
|
+
This is the predicate behind :meth:`Study.supports_calibration`. It asks the
|
|
617
|
+
same two resolvers the chart code asks — :meth:`_resolve_by_grouping` for
|
|
618
|
+
Xbar/S and :meth:`_resolve_xmr_stratification` for X/mR — rather than
|
|
619
|
+
restating when stratification happens, because that rule is not obvious:
|
|
620
|
+
Xbar/S stratify on ``by=[time]`` **only when charting the response**, so the
|
|
621
|
+
same ``by`` is refused for the response and accepted for a residual.
|
|
622
|
+
"""
|
|
623
|
+
residual = self.request.residual
|
|
624
|
+
if residual:
|
|
625
|
+
chart_type = self.request.residual_chart_type or 'X'
|
|
626
|
+
value_col = f'{"RCR" if self.request.recentered else "R"}{residual[1:]}'
|
|
627
|
+
else:
|
|
628
|
+
chart_type = self.analysis_type
|
|
629
|
+
value_col = self.spec.response_var
|
|
630
|
+
|
|
631
|
+
if chart_type in ('Xbar', 'S'):
|
|
632
|
+
_, _, stratify_by = self._resolve_by_grouping(value_col)
|
|
633
|
+
return f'stratified {chart_type}' if stratify_by else None
|
|
634
|
+
|
|
635
|
+
if chart_type in ('X', 'mR'):
|
|
636
|
+
stratify_by, collapsed_factors = self._resolve_xmr_stratification()
|
|
637
|
+
if stratify_by:
|
|
638
|
+
return 'stratified X/mR'
|
|
639
|
+
if self.request.phased and collapsed_factors:
|
|
640
|
+
return 'phased X/mR'
|
|
641
|
+
return None
|
|
642
|
+
|
|
643
|
+
# Histogram and anything else limit-free: nothing for a calibration to change.
|
|
644
|
+
return None
|
|
645
|
+
|
|
646
|
+
def _reject_calibration(self, context: str) -> None:
|
|
647
|
+
"""Raise if a calibration is set on a path that does not yet support it.
|
|
648
|
+
|
|
649
|
+
Called at the branch points that would otherwise silently ignore it. The
|
|
650
|
+
same rule, asked ahead of time, is :meth:`calibration_rejection_reason`.
|
|
651
|
+
"""
|
|
652
|
+
if self.calibration is not None:
|
|
653
|
+
raise CalibrationNotSupportedError(
|
|
654
|
+
f'calibration is not supported with {context} yet. '
|
|
655
|
+
'Apply a calibration to a non-stratified, non-phased chart '
|
|
656
|
+
'(e.g. study.execute(chart="X", by=[], calibration=cal)). '
|
|
657
|
+
'Ask first with study.supports_calibration(chart=..., by=...).',
|
|
658
|
+
context=context,
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
def _calibrated_row(self, limits_type: str, N: int | None = None) -> pd.Series:
|
|
662
|
+
"""Calibrated (center, lpl, upl) for one row, as a Series for DataFrame.apply.
|
|
663
|
+
|
|
664
|
+
Delegates to :func:`calibrated_limits`, which reuses ``calculate_limits``
|
|
665
|
+
so rounding/clamping match the data path exactly.
|
|
666
|
+
"""
|
|
667
|
+
cal = self.calibration
|
|
668
|
+
center, lims = calibrated_limits(
|
|
669
|
+
limits_type,
|
|
670
|
+
mean=cal.mean,
|
|
671
|
+
sigma=cal.sigma,
|
|
672
|
+
N=N,
|
|
673
|
+
n_sigma=self.request.n_sigma,
|
|
674
|
+
center_zero=self._calibration_center_zero,
|
|
675
|
+
round_to=self.spec.round_to,
|
|
676
|
+
)
|
|
677
|
+
return pd.Series({'center': center, 'lpl': lims['lpl'], 'upl': lims['upl']})
|
|
678
|
+
|
|
679
|
+
def _calibrated_scalar(self, limits_type: str, N: int | None = None) -> tuple[float, pd.Series]:
|
|
680
|
+
"""Calibrated (center, limits_series) for a single-stream chart (X/mR).
|
|
681
|
+
|
|
682
|
+
XmR/R take no subgroup size; ``N`` is accepted for symmetry only.
|
|
683
|
+
"""
|
|
684
|
+
cal = self.calibration
|
|
685
|
+
return calibrated_limits(
|
|
686
|
+
limits_type,
|
|
687
|
+
mean=cal.mean,
|
|
688
|
+
sigma=cal.sigma,
|
|
689
|
+
N=N,
|
|
690
|
+
n_sigma=self.request.n_sigma,
|
|
691
|
+
center_zero=self._calibration_center_zero,
|
|
692
|
+
round_to=self.spec.round_to,
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
def _apply_calibration_metadata(self, chart_data: dict) -> dict:
|
|
696
|
+
"""Tag each chart payload's metadata with the calibration provenance."""
|
|
697
|
+
if self.calibration is None:
|
|
698
|
+
return chart_data
|
|
699
|
+
cal = self.calibration
|
|
700
|
+
badge = {
|
|
701
|
+
'limits_source': 'calibration',
|
|
702
|
+
'calibration': {'label': cal.label, 'mean': cal.mean, 'sigma': cal.sigma},
|
|
703
|
+
}
|
|
704
|
+
return {
|
|
705
|
+
key: {**data, 'metadata': {**data.get('metadata', {}), **badge}}
|
|
706
|
+
for key, data in chart_data.items()
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
@staticmethod
|
|
710
|
+
def _add_residual_metadata(chart_data, residual, recentered, questions):
|
|
711
|
+
"""Add residual context to chart results. Returns new dict (no mutation)."""
|
|
712
|
+
result = {}
|
|
713
|
+
for chart_key, data in chart_data.items():
|
|
714
|
+
result[chart_key] = {
|
|
715
|
+
**data,
|
|
716
|
+
'metadata': {
|
|
717
|
+
**data.get('metadata', {}),
|
|
718
|
+
'residual_type': residual,
|
|
719
|
+
'recentered': recentered,
|
|
720
|
+
'question_answered': questions.get(residual, ''),
|
|
721
|
+
},
|
|
722
|
+
}
|
|
723
|
+
return result
|
|
724
|
+
|
|
725
|
+
def _add_beyond_limits_flag(
|
|
726
|
+
self, df: pd.DataFrame, value_col: str, lpl_col: str = 'lpl', upl_col: str = 'upl'
|
|
727
|
+
) -> pd.DataFrame:
|
|
728
|
+
"""
|
|
729
|
+
Add beyond_limits flag column.
|
|
730
|
+
|
|
731
|
+
Returns -1 for below LPL, 1 for above UPL, 0 for in control.
|
|
732
|
+
|
|
733
|
+
Pure function: doesn't modify input.
|
|
734
|
+
|
|
735
|
+
Parameters
|
|
736
|
+
----------
|
|
737
|
+
df : pd.DataFrame
|
|
738
|
+
Input data with control limits
|
|
739
|
+
value_col : str
|
|
740
|
+
Column name containing values to check
|
|
741
|
+
lpl_col : str, default 'lpl'
|
|
742
|
+
Column name for lower process limit
|
|
743
|
+
upl_col : str, default 'upl'
|
|
744
|
+
Column name for upper process limit
|
|
745
|
+
|
|
746
|
+
Returns
|
|
747
|
+
-------
|
|
748
|
+
pd.DataFrame
|
|
749
|
+
Data with added 'beyond_limits' column
|
|
750
|
+
"""
|
|
751
|
+
result = df.copy()
|
|
752
|
+
|
|
753
|
+
# Vectorized form of detect_beyond_limits, which stays the scalar reference (the
|
|
754
|
+
# Bishop validator exercises it, so it should not become a check of this against
|
|
755
|
+
# itself). One Python call per *observation* was the dominant cost of execute() on
|
|
756
|
+
# X/mR charts — 2.14s at 1M rows, versus 0.002s here.
|
|
757
|
+
#
|
|
758
|
+
# The nesting order matters: the scalar rule tests `x < lpl` FIRST, then `x > upl`.
|
|
759
|
+
# NaN compares False both ways in numpy exactly as in Python, so a NaN observation
|
|
760
|
+
# or a NaN limit still yields 0.
|
|
761
|
+
values = result[value_col].to_numpy()
|
|
762
|
+
lower = result[lpl_col].to_numpy()
|
|
763
|
+
upper = result[upl_col].to_numpy()
|
|
764
|
+
result['beyond_limits'] = np.where(values < lower, -1, np.where(values > upper, 1, 0))
|
|
765
|
+
|
|
766
|
+
return result
|
|
767
|
+
|
|
768
|
+
def _determine_n_to_use(self, df: pd.DataFrame, n_col: str = 'n') -> tuple[str, int]:
|
|
769
|
+
"""
|
|
770
|
+
Determine if subgroup sizes are constant or variable.
|
|
771
|
+
|
|
772
|
+
Parameters
|
|
773
|
+
----------
|
|
774
|
+
df : pd.DataFrame
|
|
775
|
+
Input data with subgroup size column
|
|
776
|
+
n_col : str, default 'n'
|
|
777
|
+
Column name containing subgroup sizes
|
|
778
|
+
|
|
779
|
+
Returns
|
|
780
|
+
-------
|
|
781
|
+
tuple[str, int]
|
|
782
|
+
(n_to_use, max_n) where:
|
|
783
|
+
- n_to_use: 'N' if constant sizes, 'n' if variable
|
|
784
|
+
- max_n: maximum subgroup size
|
|
785
|
+
|
|
786
|
+
Examples
|
|
787
|
+
--------
|
|
788
|
+
>>> n_to_use, max_n = self._determine_n_to_use(out)
|
|
789
|
+
>>> # Use max_n for statistics, n_to_use for per-row calculations
|
|
790
|
+
"""
|
|
791
|
+
n_max = df[n_col].max()
|
|
792
|
+
n_to_use = 'N' if df[n_col].eq(n_max).all() else 'n'
|
|
793
|
+
|
|
794
|
+
logger.debug('Analysis using %s for calculations (Scenario: %s)', n_to_use, 1 if n_to_use == 'N' else 2)
|
|
795
|
+
|
|
796
|
+
return n_to_use, n_max
|
|
797
|
+
|
|
798
|
+
def _residual_grain(self, value_col: str) -> list[str]:
|
|
799
|
+
"""Return the cell-grid columns at which to compute Bishop's grand mean
|
|
800
|
+
for an Xbar chart on this column.
|
|
801
|
+
|
|
802
|
+
Each VAS residual is recentered around a baseline computed at a specific
|
|
803
|
+
grain (see `analysis_dataset.py:364-371` and `residual_calculator.calculate_r6_residuals`). The
|
|
804
|
+
canonical grand mean of an RCRk chart averages over that grain — equal
|
|
805
|
+
weight per cell — yielding Bishop's unweighted center line.
|
|
806
|
+
|
|
807
|
+
Grain table (None for non-residuals; falls through to other logic):
|
|
808
|
+
R1, R2, R3 -> [rsg_var_name, time_var] (full cell grid)
|
|
809
|
+
R4 -> [rsg_var_name] (time effect removed)
|
|
810
|
+
R5, R6 -> [rsg_var_name] (factor effects live at rsg level)
|
|
811
|
+
|
|
812
|
+
Returns an empty list when the column isn't a residual or the spec
|
|
813
|
+
lacks the required factors/time — caller falls back to `df.mean()`.
|
|
814
|
+
"""
|
|
815
|
+
if value_col is None:
|
|
816
|
+
return []
|
|
817
|
+
spec = self.spec
|
|
818
|
+
base = value_col.upper()
|
|
819
|
+
if base.startswith('RCR'):
|
|
820
|
+
base = 'R' + base[3:]
|
|
821
|
+
if not (base.startswith('R') and len(base) >= 2 and base[1].isdigit()):
|
|
822
|
+
return []
|
|
823
|
+
# R5/R6 are factor-effect residuals; R4 is the time-effect residual
|
|
824
|
+
# (time component removed). For our SDS 3 validation only R3 and R6 are
|
|
825
|
+
# exercised; R4/R5 grains are inferred from the recentering structure.
|
|
826
|
+
if base in ('R5', 'R6'):
|
|
827
|
+
return [spec.rsg_var_name] if spec.rsg_var_name else []
|
|
828
|
+
if base == 'R4':
|
|
829
|
+
return [spec.rsg_var_name] if spec.rsg_var_name else []
|
|
830
|
+
# R1, R2, R3: full (rsg x time) cell grid
|
|
831
|
+
grain = []
|
|
832
|
+
if spec.rsg_var_name:
|
|
833
|
+
grain.append(spec.rsg_var_name)
|
|
834
|
+
if spec.time_var:
|
|
835
|
+
grain.append(spec.time_var)
|
|
836
|
+
return grain
|
|
837
|
+
|
|
838
|
+
@staticmethod
|
|
839
|
+
def _resolve_limits_column(value_col: str, df: pd.DataFrame) -> str:
|
|
840
|
+
"""Return the column to use for within-group std in limit calculations.
|
|
841
|
+
|
|
842
|
+
For effect-carrying residuals (R1/R3/R4/R5 and RCR variants), use R2 instead.
|
|
843
|
+
Non-R2 residuals retain structural effects (factor, time, interaction)
|
|
844
|
+
whose within-group std inflates limits. R2 (within-cell noise) is the
|
|
845
|
+
correct dispersion basis per Bishop.
|
|
846
|
+
|
|
847
|
+
Always uses plain R2 (not RCR2) because recentered residuals add
|
|
848
|
+
cell-specific offsets that inflate within-group std when groups span
|
|
849
|
+
multiple cells.
|
|
850
|
+
"""
|
|
851
|
+
_EFFECT_RESIDUALS = {'R1', 'R3', 'R4', 'R5', 'RCR1', 'RCR3', 'RCR4', 'RCR5'}
|
|
852
|
+
if value_col is not None and value_col.upper() in _EFFECT_RESIDUALS and 'R2' in df.columns:
|
|
853
|
+
return 'R2'
|
|
854
|
+
return value_col
|
|
855
|
+
|
|
856
|
+
def _resolve_by_grouping(self, value_col: str) -> tuple[list[str], str | None, list[str]]:
|
|
857
|
+
"""
|
|
858
|
+
Resolve groupby columns and pre-calculated Ybar column based on `by` parameter.
|
|
859
|
+
|
|
860
|
+
The `by` parameter controls aggregation level for Xbar/S charts.
|
|
861
|
+
When possible, we use pre-calculated Ybar columns from the analytic dataset.
|
|
862
|
+
|
|
863
|
+
Parameters
|
|
864
|
+
----------
|
|
865
|
+
value_col : str
|
|
866
|
+
Column being charted (response_var or residual column)
|
|
867
|
+
|
|
868
|
+
Returns
|
|
869
|
+
-------
|
|
870
|
+
tuple[list[str], str | None, list[str]]
|
|
871
|
+
(groupby_cols, ybar_col, stratify_by) where:
|
|
872
|
+
- groupby_cols: columns to group by (empty list for collapse all)
|
|
873
|
+
- ybar_col: pre-calculated Ybar column to use, or None if must aggregate
|
|
874
|
+
- stratify_by: columns to stratify by (separate charts per combo),
|
|
875
|
+
empty list if no stratification
|
|
876
|
+
|
|
877
|
+
Examples
|
|
878
|
+
--------
|
|
879
|
+
>>> groupby_cols, ybar_col, stratify_by = self._resolve_by_grouping('y')
|
|
880
|
+
>>> # by=[] (normalized to by=None) -> ([rsg, time_var], 'Ybar_kt', [])
|
|
881
|
+
>>> # - cell-level aggregation when factors+time exist
|
|
882
|
+
>>> # by=['factor1','factor2'] -> (['rsg'], 'Ybar_k', []) - use factor means
|
|
883
|
+
>>> # by=['time_var'] with factors -> ([time_var], 'Ybar_t', [rsg_var_name])
|
|
884
|
+
>>> # NOTE: ybar_col is None for residual value_col (no pre-cached mean).
|
|
885
|
+
"""
|
|
886
|
+
spec = self.spec
|
|
887
|
+
by = self.request.by
|
|
888
|
+
# Normalize tuple to list for internal comparison
|
|
889
|
+
by = list(by) if by is not None else None
|
|
890
|
+
rsg_vars = spec.rsg_vars_list
|
|
891
|
+
time_var = spec.time_var
|
|
892
|
+
|
|
893
|
+
# Determine if we're charting response (can use pre-calculated) or residual
|
|
894
|
+
is_response = value_col == spec.response_var
|
|
895
|
+
|
|
896
|
+
# Xbar/S: by=[] normalizes to by=None (Kt-level aggregation).
|
|
897
|
+
# X/mR handles by=[] directly in _calculate_xmr() as single-stream.
|
|
898
|
+
# This split is intentional — `by` means "aggregate by" for Xbar/S
|
|
899
|
+
# and "stratify by" for X/mR, matching each chart type's semantics.
|
|
900
|
+
if by == []:
|
|
901
|
+
by = None
|
|
902
|
+
|
|
903
|
+
# Default: by=None means use cell_key level (factors + time)
|
|
904
|
+
if by is None:
|
|
905
|
+
if time_var and rsg_vars:
|
|
906
|
+
# Factors + time (Kt level) - matches cell_key
|
|
907
|
+
groupby_cols = rsg_vars + [time_var]
|
|
908
|
+
ybar_col = 'Ybar_kt' if is_response else None
|
|
909
|
+
elif rsg_vars:
|
|
910
|
+
# Factors only (no time)
|
|
911
|
+
groupby_cols = [spec.rsg_var_name]
|
|
912
|
+
ybar_col = 'Ybar_k' if is_response else None
|
|
913
|
+
elif time_var:
|
|
914
|
+
# Time only (no factors)
|
|
915
|
+
groupby_cols = [time_var]
|
|
916
|
+
ybar_col = 'Ybar_t' if is_response else None
|
|
917
|
+
else:
|
|
918
|
+
# No grouping
|
|
919
|
+
groupby_cols = []
|
|
920
|
+
ybar_col = None
|
|
921
|
+
return groupby_cols, ybar_col, []
|
|
922
|
+
|
|
923
|
+
# Check if by matches known aggregation levels for Ybar optimization
|
|
924
|
+
by_set = set(by)
|
|
925
|
+
rsg_set = set(rsg_vars)
|
|
926
|
+
|
|
927
|
+
# by == all factors (rsg_key level) -> use Ybar_k optimization
|
|
928
|
+
# Only if order matches rsg_vars; otherwise preserve user's order
|
|
929
|
+
if by_set == rsg_set and list(by) == rsg_vars:
|
|
930
|
+
groupby_cols = [spec.rsg_var_name]
|
|
931
|
+
ybar_col = 'Ybar_k' if is_response else None
|
|
932
|
+
return groupby_cols, ybar_col, []
|
|
933
|
+
|
|
934
|
+
# by == [time_var] only -> use Ybar_t
|
|
935
|
+
# When factors exist and charting response, also stratify by factor combos
|
|
936
|
+
if by_set == {time_var}:
|
|
937
|
+
groupby_cols = [time_var]
|
|
938
|
+
ybar_col = 'Ybar_t' if is_response else None
|
|
939
|
+
stratify_by = [spec.rsg_var_name] if (rsg_vars and is_response) else []
|
|
940
|
+
return groupby_cols, ybar_col, stratify_by
|
|
941
|
+
|
|
942
|
+
# by == all factors + time (cell_key level) -> use Ybar_kt
|
|
943
|
+
cell_key_vars = rsg_set | ({time_var} if time_var else set())
|
|
944
|
+
if by_set == cell_key_vars:
|
|
945
|
+
# Group by all factors + time
|
|
946
|
+
groupby_cols = rsg_vars + [time_var] if time_var else rsg_vars
|
|
947
|
+
ybar_col = 'Ybar_kt' if is_response else None
|
|
948
|
+
return groupby_cols, ybar_col, []
|
|
949
|
+
|
|
950
|
+
# Partial subset - must aggregate at runtime
|
|
951
|
+
# Use the by columns directly
|
|
952
|
+
groupby_cols = list(by)
|
|
953
|
+
return groupby_cols, None, []
|
|
954
|
+
|
|
955
|
+
def _calculate_lane_boundaries(self, df: pd.DataFrame, collapsed_vars: list[str]) -> list[dict]:
|
|
956
|
+
"""
|
|
957
|
+
Calculate lane boundaries where collapsed factors change.
|
|
958
|
+
|
|
959
|
+
Lane boundaries are positions in the data where a factor that was
|
|
960
|
+
"collapsed" (not in `by`) changes value. These are rendered as
|
|
961
|
+
vertical dashed lines on X charts.
|
|
962
|
+
|
|
963
|
+
Parameters
|
|
964
|
+
----------
|
|
965
|
+
df : pd.DataFrame
|
|
966
|
+
Data in display order (already sorted)
|
|
967
|
+
collapsed_vars : list[str]
|
|
968
|
+
Variables that were collapsed (not in `by`)
|
|
969
|
+
|
|
970
|
+
Returns
|
|
971
|
+
-------
|
|
972
|
+
list[dict]
|
|
973
|
+
List of boundary dicts with 'position' (x index) and 'label' (factor value)
|
|
974
|
+
|
|
975
|
+
Examples
|
|
976
|
+
--------
|
|
977
|
+
>>> # If by=['factor1'] and data has factor1='A' with factor2 changing X→Y
|
|
978
|
+
>>> boundaries = self._calculate_lane_boundaries(df, ['factor2'])
|
|
979
|
+
>>> # Returns: [{'position': 5, 'label': 'Y', 'variable': 'factor2'}]
|
|
980
|
+
"""
|
|
981
|
+
if not collapsed_vars or df.empty:
|
|
982
|
+
return []
|
|
983
|
+
|
|
984
|
+
# Combine column-wise rather than with `.agg('_'.join, axis=1)`, which ran once per
|
|
985
|
+
# row (1.05s at 1M). Deliberately NOT encode_rsg_series: that renders a datetime as
|
|
986
|
+
# str(Timestamp) ('2024-01-01 00:00:00') where DataFrame.astype(str) gives
|
|
987
|
+
# '2024-01-01'. These are display labels for a chart divider, not stratum identity,
|
|
988
|
+
# so they keep the astype rendering — do not "unify" the two.
|
|
989
|
+
parts = [df[col].astype(str) for col in collapsed_vars]
|
|
990
|
+
combined = parts[0] if len(parts) == 1 else reduce(lambda left, right: left + '_' + right, parts)
|
|
991
|
+
|
|
992
|
+
# Positional comparison against the previous row. The old form built a boolean mask,
|
|
993
|
+
# took df.index[mask], dropped the first entry, then called df.index.get_loc() per
|
|
994
|
+
# boundary — and get_loc is not O(1) on these frames (~1.3ms a call, ~1.8s total).
|
|
995
|
+
# Comparing the array to itself shifted gives the 0-based positions directly.
|
|
996
|
+
labels = combined.to_numpy()
|
|
997
|
+
changed = np.empty(len(labels), dtype=bool)
|
|
998
|
+
changed[0] = False # the first row is a start, not a boundary
|
|
999
|
+
changed[1:] = labels[1:] != labels[:-1]
|
|
1000
|
+
|
|
1001
|
+
return [
|
|
1002
|
+
{
|
|
1003
|
+
'position': int(pos), # 0-based position in the sorted data
|
|
1004
|
+
'label': labels[pos],
|
|
1005
|
+
'variables': collapsed_vars,
|
|
1006
|
+
}
|
|
1007
|
+
for pos in np.flatnonzero(changed)
|
|
1008
|
+
]
|
|
1009
|
+
|
|
1010
|
+
def _build_output_columns(self, df: pd.DataFrame, value_cols: list[str]) -> pd.DataFrame:
|
|
1011
|
+
"""
|
|
1012
|
+
Select output columns with appropriate time/grouping columns.
|
|
1013
|
+
|
|
1014
|
+
Handles the pattern:
|
|
1015
|
+
- If has_time and has_grouping: [time, rsg, ...values, obs_id]
|
|
1016
|
+
- If has_time only: [time, ...values, obs_id]
|
|
1017
|
+
- If has_grouping only: [x, rsg, ...values, obs_id] (with generated x)
|
|
1018
|
+
- Otherwise: [x, ...values, obs_id] (with generated x)
|
|
1019
|
+
|
|
1020
|
+
Always includes obs_id (if available) for traceability of violations.
|
|
1021
|
+
|
|
1022
|
+
Parameters
|
|
1023
|
+
----------
|
|
1024
|
+
df : pd.DataFrame
|
|
1025
|
+
Input data
|
|
1026
|
+
value_cols : list[str]
|
|
1027
|
+
Columns to keep (e.g., ['mean', 'lpl', 'upl', 'beyond_limits'])
|
|
1028
|
+
|
|
1029
|
+
Returns
|
|
1030
|
+
-------
|
|
1031
|
+
pd.DataFrame
|
|
1032
|
+
Data with selected columns, rounded
|
|
1033
|
+
|
|
1034
|
+
Notes
|
|
1035
|
+
-----
|
|
1036
|
+
The obs_id column enables tracing violations back to original data:
|
|
1037
|
+
- Find violations: df[df['beyond_limits'] != 0]
|
|
1038
|
+
- Get obs_ids: violation_ids = df['obs_id'].tolist()
|
|
1039
|
+
- Trace to raw data: raw_df[raw_df['obs_id'].isin(violation_ids)]
|
|
1040
|
+
|
|
1041
|
+
Examples
|
|
1042
|
+
--------
|
|
1043
|
+
>>> out = self._build_output_columns(
|
|
1044
|
+
... df=out,
|
|
1045
|
+
... value_cols=[spec.response_var, 'mean', 'lpl', 'upl', 'beyond_limits']
|
|
1046
|
+
... )
|
|
1047
|
+
"""
|
|
1048
|
+
result = df.copy()
|
|
1049
|
+
cols_to_keep = value_cols.copy()
|
|
1050
|
+
|
|
1051
|
+
if self.spec.has_time:
|
|
1052
|
+
if self.spec.has_grouping:
|
|
1053
|
+
cols_to_keep.insert(0, self.spec.rsg_var_name)
|
|
1054
|
+
cols_to_keep.insert(0, self.spec.time_var)
|
|
1055
|
+
else:
|
|
1056
|
+
cols_to_keep.insert(0, self.spec.time_var)
|
|
1057
|
+
else:
|
|
1058
|
+
# No explicit time variable specified - use implicit ordering
|
|
1059
|
+
if self.spec.has_grouping:
|
|
1060
|
+
# Per-group sequence number for grouped data
|
|
1061
|
+
result['x'] = result.groupby(self.spec.rsg_var_name, observed=True).cumcount() + 1
|
|
1062
|
+
cols_to_keep.insert(0, self.spec.rsg_var_name)
|
|
1063
|
+
cols_to_keep.insert(0, 'x')
|
|
1064
|
+
else:
|
|
1065
|
+
# Use obs_id as implicit time for single condition over time (SDS 4)
|
|
1066
|
+
# Rationale: Wheeler's X chart assumes temporal ordering, and obs_id
|
|
1067
|
+
# provides that ordering from the original data sequence.
|
|
1068
|
+
# See: ProcessBehavior.formulate() docstring for full explanation.
|
|
1069
|
+
cols_to_keep.insert(0, 'obs_id')
|
|
1070
|
+
|
|
1071
|
+
# Always include obs_id for traceability (if available)
|
|
1072
|
+
# This enables linking violations back to original data
|
|
1073
|
+
if 'obs_id' in result.columns and 'obs_id' not in cols_to_keep:
|
|
1074
|
+
cols_to_keep.append('obs_id')
|
|
1075
|
+
|
|
1076
|
+
return result[cols_to_keep].round(self.spec.round_to)
|
|
1077
|
+
|
|
1078
|
+
# =========================================================================
|
|
1079
|
+
# Chart Calculation Methods (Strategy Pattern)
|
|
1080
|
+
# =========================================================================
|
|
1081
|
+
|
|
1082
|
+
def _calculate_xbar( # noqa: C901
|
|
1083
|
+
self, value_col: str | None = None, _return_intermediates: bool = False
|
|
1084
|
+
) -> dict:
|
|
1085
|
+
"""
|
|
1086
|
+
Calculate Xbar (mean) chart statistics.
|
|
1087
|
+
|
|
1088
|
+
Parameters
|
|
1089
|
+
----------
|
|
1090
|
+
value_col : str, optional
|
|
1091
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
1092
|
+
Pass a residual column (R2, R5, etc.) for residual charts.
|
|
1093
|
+
_return_intermediates : bool, optional
|
|
1094
|
+
If True, includes '_intermediates' key with values needed by
|
|
1095
|
+
_calculate_s() to avoid redundant computation. Used internally
|
|
1096
|
+
by _calculate_xbar_s() for companion chart calculation.
|
|
1097
|
+
|
|
1098
|
+
Returns
|
|
1099
|
+
-------
|
|
1100
|
+
dict
|
|
1101
|
+
Chart data: {'Xbar': {'data': df, 'statistics': dict, 'metadata': dict}}
|
|
1102
|
+
If _return_intermediates=True, also includes '_intermediates' key.
|
|
1103
|
+
|
|
1104
|
+
Logic moved from Xbar.calculate_statistics()
|
|
1105
|
+
"""
|
|
1106
|
+
spec = self.spec
|
|
1107
|
+
if value_col is None:
|
|
1108
|
+
value_col = spec.response_var
|
|
1109
|
+
|
|
1110
|
+
result = {}
|
|
1111
|
+
statistics = {}
|
|
1112
|
+
df = self._source_frame().copy()
|
|
1113
|
+
|
|
1114
|
+
logger.debug('In calculate statistics XbarS')
|
|
1115
|
+
logger.debug('Dataframe has columns: %s', df.columns.to_list())
|
|
1116
|
+
logger.debug('Dataframe head:\n%s', df.head(10))
|
|
1117
|
+
|
|
1118
|
+
# Resolve grouping based on `by` parameter
|
|
1119
|
+
groupby_cols, ybar_col, stratify_by = self._resolve_by_grouping(value_col)
|
|
1120
|
+
|
|
1121
|
+
# Stratified path: separate chart per factor combination
|
|
1122
|
+
if stratify_by:
|
|
1123
|
+
self._reject_calibration('stratified Xbar')
|
|
1124
|
+
return self._calculate_xbar_stratified(
|
|
1125
|
+
value_col,
|
|
1126
|
+
groupby_cols,
|
|
1127
|
+
stratify_by,
|
|
1128
|
+
_return_intermediates=_return_intermediates,
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
_limits_col = self._resolve_limits_column(value_col, df)
|
|
1132
|
+
|
|
1133
|
+
# Bishop VAS grand mean: equal weight per cell at the residual's natural
|
|
1134
|
+
# grain, regardless of cell N. The unweighted form is canonical; weighted
|
|
1135
|
+
# vs unweighted differ only on unbalanced designs, and Bishop's principle
|
|
1136
|
+
# is that the practical difference is negligible -- the methodology still
|
|
1137
|
+
# requires the unweighted form. Each residual is recentered at a specific
|
|
1138
|
+
# grain (analysis_dataset.py:364-371; residual_calculator.calculate_r6_residuals for R6), and the grand
|
|
1139
|
+
# mean averages over that grain:
|
|
1140
|
+
# R3 (interaction) -> (rsg, time) full cell grid
|
|
1141
|
+
# R4 (time effect) -> (rsg) factor grain (time component removed)
|
|
1142
|
+
# R5, R6 (factor effects) -> (time) or (rsg) depending on what's removed
|
|
1143
|
+
# In practice the only residuals exercised by Bishop's validation are R3
|
|
1144
|
+
# (full cell grid) and R5/R6 (factor grain).
|
|
1145
|
+
if value_col == spec.response_var and 'Ybar' in df.columns:
|
|
1146
|
+
_Ybar = df['Ybar'].iloc[0]
|
|
1147
|
+
else:
|
|
1148
|
+
grain_cols = self._residual_grain(value_col)
|
|
1149
|
+
if grain_cols and value_col in df.columns:
|
|
1150
|
+
_Ybar = df.groupby(grain_cols, observed=True)[value_col].mean().mean()
|
|
1151
|
+
else:
|
|
1152
|
+
_Ybar = df[value_col].mean()
|
|
1153
|
+
|
|
1154
|
+
# Handle by=[] (collapse all) - single point chart
|
|
1155
|
+
if groupby_cols == []:
|
|
1156
|
+
# Single aggregation across all data
|
|
1157
|
+
_S = df[_limits_col].std()
|
|
1158
|
+
_N = len(df)
|
|
1159
|
+
out_dict = {'group': ['All'], 'xbar': [_Ybar], 's': [_S], 'n': [_N], 'N': [_N]}
|
|
1160
|
+
if _limits_col != value_col:
|
|
1161
|
+
out_dict['s_value'] = [df[value_col].std()]
|
|
1162
|
+
out = pd.DataFrame(out_dict)
|
|
1163
|
+
else:
|
|
1164
|
+
# Group by specified columns
|
|
1165
|
+
agg_dict = {
|
|
1166
|
+
's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
|
|
1167
|
+
'mean': pd.NamedAgg(column=value_col, aggfunc='mean'),
|
|
1168
|
+
# Count on response_var (not value_col) to avoid NaN issues with residuals
|
|
1169
|
+
'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
|
|
1170
|
+
}
|
|
1171
|
+
if _limits_col != value_col:
|
|
1172
|
+
agg_dict['s_value'] = pd.NamedAgg(column=value_col, aggfunc='std')
|
|
1173
|
+
out = df.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
|
|
1174
|
+
|
|
1175
|
+
# If we have pre-calculated Ybar, use it for xbar values
|
|
1176
|
+
if ybar_col and ybar_col in df.columns:
|
|
1177
|
+
# Get unique Ybar values per group
|
|
1178
|
+
ybar_by_group = df.groupby(groupby_cols, observed=True)[ybar_col].first().reset_index()
|
|
1179
|
+
out = out.merge(ybar_by_group, on=groupby_cols, how='left')
|
|
1180
|
+
out['xbar'] = out[ybar_col]
|
|
1181
|
+
out = out.drop(columns=[ybar_col, 'mean'], errors='ignore')
|
|
1182
|
+
else:
|
|
1183
|
+
# Rename columns for consistency: mean→xbar (values)
|
|
1184
|
+
out = out.rename(columns={'mean': 'xbar'})
|
|
1185
|
+
|
|
1186
|
+
_N = out['n'].max()
|
|
1187
|
+
out['N'] = _N
|
|
1188
|
+
|
|
1189
|
+
# Filter out groups with n=1 (can't compute c4 for variance estimation)
|
|
1190
|
+
# Xbar charts require n >= 2 for within-group variance.
|
|
1191
|
+
# Xbar stays in valid_charts on ADS 2 — the chart family is valid there
|
|
1192
|
+
# (Xbar of a residual, or of a pooled subgroup, computes fine); it is
|
|
1193
|
+
# only charting *the response* that has nothing to subgroup. See
|
|
1194
|
+
# Study._response_pair_problem, which reports that in advance.
|
|
1195
|
+
mask_n1 = out['n'].eq(1)
|
|
1196
|
+
if mask_n1.any():
|
|
1197
|
+
n_filtered = mask_n1.sum()
|
|
1198
|
+
logger.info(f'Filtered {n_filtered} subgroup(s) with n=1 from Xbar calculation')
|
|
1199
|
+
out = out[~mask_n1].copy()
|
|
1200
|
+
|
|
1201
|
+
# Handle case where no subgroups have >1 observation
|
|
1202
|
+
if out.shape[0] == 0:
|
|
1203
|
+
sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
|
|
1204
|
+
_raise_no_replicated_subgroups('Xbar', sds)
|
|
1205
|
+
|
|
1206
|
+
# Use Bishop VAS grand mean (mean of cell means on value_col) as center
|
|
1207
|
+
_Xbar = _Ybar
|
|
1208
|
+
_S = out['s'].mean()
|
|
1209
|
+
_N = out['n'].max()
|
|
1210
|
+
if 'N' not in out.columns:
|
|
1211
|
+
out['N'] = _N
|
|
1212
|
+
|
|
1213
|
+
# Determine if subgroup sizes are constant or variable
|
|
1214
|
+
n_to_use, n_max = self._determine_n_to_use(out)
|
|
1215
|
+
|
|
1216
|
+
# Override n_to_use if n_mode="average"
|
|
1217
|
+
n_mode = self.request.n_mode
|
|
1218
|
+
n_avg = None
|
|
1219
|
+
if n_mode == 'average':
|
|
1220
|
+
n_avg = out['n'].mean()
|
|
1221
|
+
out['N'] = n_avg # overwrite N column with average
|
|
1222
|
+
n_to_use = 'N' # force constant-N path
|
|
1223
|
+
|
|
1224
|
+
# CALCULATE XBAR
|
|
1225
|
+
xbar = out.copy()
|
|
1226
|
+
if self.calibration is not None:
|
|
1227
|
+
# Standards-given Xbar: center=mean, mean ± n_sigma·sigma/√N (per-row N).
|
|
1228
|
+
xbar[['center', 'lpl', 'upl']] = xbar.apply(
|
|
1229
|
+
lambda row: self._calibrated_row('Xbar', N=row[n_to_use]),
|
|
1230
|
+
axis=1,
|
|
1231
|
+
)
|
|
1232
|
+
else:
|
|
1233
|
+
xbar['center'] = _Xbar # Add center column for Xbar chart
|
|
1234
|
+
xbar[['lpl', 'upl']] = calculate_limits_vectorized(
|
|
1235
|
+
'Xbar',
|
|
1236
|
+
mean=xbar['center'],
|
|
1237
|
+
sd=_S,
|
|
1238
|
+
N=xbar[n_to_use],
|
|
1239
|
+
sigma_multiplier=self.request.n_sigma,
|
|
1240
|
+
)
|
|
1241
|
+
|
|
1242
|
+
# Detect beyond limits signals
|
|
1243
|
+
xbar = self._add_beyond_limits_flag(xbar, value_col='xbar')
|
|
1244
|
+
xbar = xbar.round(spec.round_to)
|
|
1245
|
+
|
|
1246
|
+
_center_stat = xbar['center'].iloc[0] if self.calibration is not None else _Xbar
|
|
1247
|
+
statistics['center'] = round(_center_stat, spec.round_to)
|
|
1248
|
+
if n_to_use == 'N':
|
|
1249
|
+
statistics['N'] = round(n_avg, spec.round_to) if n_avg is not None else _N
|
|
1250
|
+
statistics['upl'] = xbar['upl'].max()
|
|
1251
|
+
statistics['lpl'] = xbar['lpl'].max()
|
|
1252
|
+
else:
|
|
1253
|
+
statistics['N'] = None
|
|
1254
|
+
statistics['lpl'] = None
|
|
1255
|
+
statistics['upl'] = None
|
|
1256
|
+
statistics['limits_vary'] = True
|
|
1257
|
+
|
|
1258
|
+
# Determine the grouping column for output
|
|
1259
|
+
by = self.request.by
|
|
1260
|
+
if by is not None and len(by) > 0:
|
|
1261
|
+
# Explicit by list
|
|
1262
|
+
if len(by) == 1:
|
|
1263
|
+
# Single factor: preserve original column name for axis labeling.
|
|
1264
|
+
# Cast to string so Plotly treats coded factors (e.g., 1,2,3) as categories.
|
|
1265
|
+
xbar[groupby_cols[0]] = xbar[groupby_cols[0]].astype(str)
|
|
1266
|
+
group_col = groupby_cols[0]
|
|
1267
|
+
else:
|
|
1268
|
+
xbar['subgroup'] = xbar[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1269
|
+
group_col = 'subgroup'
|
|
1270
|
+
elif len(groupby_cols) > 1:
|
|
1271
|
+
# by=None - create 'group' from multiple columns (original behavior)
|
|
1272
|
+
xbar['group'] = xbar[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1273
|
+
group_col = 'group'
|
|
1274
|
+
elif groupby_cols:
|
|
1275
|
+
# Single groupby column
|
|
1276
|
+
group_col = groupby_cols[0]
|
|
1277
|
+
else:
|
|
1278
|
+
group_col = None
|
|
1279
|
+
|
|
1280
|
+
cols_to_keep = [group_col, 'xbar', 'center', 'lpl', 'upl', 'beyond_limits']
|
|
1281
|
+
cols_to_keep = [c for c in cols_to_keep if c in xbar.columns]
|
|
1282
|
+
xbar = xbar[cols_to_keep]
|
|
1283
|
+
result['Xbar'] = {
|
|
1284
|
+
'data': xbar,
|
|
1285
|
+
'statistics': statistics,
|
|
1286
|
+
'metadata': {
|
|
1287
|
+
'chart_type': 'Xbar',
|
|
1288
|
+
'value_col': 'xbar',
|
|
1289
|
+
'center_col': 'center',
|
|
1290
|
+
'n_sigma': self.request.n_sigma,
|
|
1291
|
+
'n_mode': n_mode,
|
|
1292
|
+
**({'n_avg': round(n_avg, spec.round_to)} if n_avg is not None else {}),
|
|
1293
|
+
},
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
# Optionally include intermediates for companion calculation
|
|
1297
|
+
if _return_intermediates:
|
|
1298
|
+
result['_intermediates'] = {
|
|
1299
|
+
'_S': _S,
|
|
1300
|
+
'n_to_use': n_to_use,
|
|
1301
|
+
'n_max': n_max,
|
|
1302
|
+
'groupby_cols': groupby_cols,
|
|
1303
|
+
'group_col': group_col,
|
|
1304
|
+
'out': out, # Pre-aggregated DataFrame with s, n, mean columns
|
|
1305
|
+
'n_avg': n_avg,
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
return result
|
|
1309
|
+
|
|
1310
|
+
def _calculate_xbar_stratified(
|
|
1311
|
+
self,
|
|
1312
|
+
value_col: str,
|
|
1313
|
+
groupby_cols: list[str],
|
|
1314
|
+
stratify_by: list[str],
|
|
1315
|
+
_return_intermediates: bool = False,
|
|
1316
|
+
) -> dict:
|
|
1317
|
+
"""
|
|
1318
|
+
Stratified Xbar chart: one chart per factor combination, time on x-axis.
|
|
1319
|
+
|
|
1320
|
+
Each stratum computes its own Sbar and limits independently.
|
|
1321
|
+
|
|
1322
|
+
Parameters
|
|
1323
|
+
----------
|
|
1324
|
+
value_col : str
|
|
1325
|
+
Column to chart (response variable).
|
|
1326
|
+
groupby_cols : list[str]
|
|
1327
|
+
Columns to group by within each stratum (typically [time_var]).
|
|
1328
|
+
stratify_by : list[str]
|
|
1329
|
+
Columns defining strata (typically [rsg_var_name]).
|
|
1330
|
+
_return_intermediates : bool
|
|
1331
|
+
If True, include intermediates for companion S chart.
|
|
1332
|
+
|
|
1333
|
+
Returns
|
|
1334
|
+
-------
|
|
1335
|
+
dict
|
|
1336
|
+
Chart data with 'strata' key for stratified output.
|
|
1337
|
+
"""
|
|
1338
|
+
spec = self.spec
|
|
1339
|
+
df = self._source_frame().copy()
|
|
1340
|
+
result = {}
|
|
1341
|
+
|
|
1342
|
+
# Determine stratification column
|
|
1343
|
+
if len(stratify_by) == 1:
|
|
1344
|
+
stratify_col = stratify_by[0]
|
|
1345
|
+
else:
|
|
1346
|
+
df['_stratify_key'] = encode_rsg_series(df, stratify_by)
|
|
1347
|
+
stratify_col = '_stratify_key'
|
|
1348
|
+
|
|
1349
|
+
_limits_col = self._resolve_limits_column(value_col, df)
|
|
1350
|
+
|
|
1351
|
+
strata = df[stratify_col].unique().tolist()
|
|
1352
|
+
all_xbar_frames = []
|
|
1353
|
+
chart_statistics = {}
|
|
1354
|
+
intermediates_per_stratum = {}
|
|
1355
|
+
# Strata with no replicated subgroup. Recorded rather than silently skipped: a
|
|
1356
|
+
# stratum published in `strata` but absent from `statistics`/`data` fails
|
|
1357
|
+
# result.focus(), which is the contract _split_strata_by_sufficiency exists to
|
|
1358
|
+
# keep on the X/mR path. Xbar/S never adopted it.
|
|
1359
|
+
insufficient_strata: list = []
|
|
1360
|
+
|
|
1361
|
+
for stratum in strata:
|
|
1362
|
+
sdf = df[df[stratify_col] == stratum].copy()
|
|
1363
|
+
|
|
1364
|
+
# Group by time within this stratum
|
|
1365
|
+
# Note: we always compute mean directly from filtered data, NOT using
|
|
1366
|
+
# Ybar_t (which is the marginal time mean across ALL factors).
|
|
1367
|
+
agg_dict = {
|
|
1368
|
+
's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
|
|
1369
|
+
'xbar': pd.NamedAgg(column=value_col, aggfunc='mean'),
|
|
1370
|
+
'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
|
|
1371
|
+
}
|
|
1372
|
+
if _limits_col != value_col:
|
|
1373
|
+
agg_dict['s_value'] = pd.NamedAgg(column=value_col, aggfunc='std')
|
|
1374
|
+
out = sdf.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
|
|
1375
|
+
|
|
1376
|
+
out['N'] = out['n'].max()
|
|
1377
|
+
|
|
1378
|
+
# Filter subgroups with n=1
|
|
1379
|
+
mask_n1 = out['n'].eq(1)
|
|
1380
|
+
if mask_n1.any():
|
|
1381
|
+
out = out[~mask_n1].copy()
|
|
1382
|
+
|
|
1383
|
+
if out.shape[0] == 0:
|
|
1384
|
+
insufficient_strata.append(stratum)
|
|
1385
|
+
continue
|
|
1386
|
+
|
|
1387
|
+
# Per-stratum statistics
|
|
1388
|
+
_Xbar = out['xbar'].mean()
|
|
1389
|
+
_S = out['s'].mean()
|
|
1390
|
+
n_to_use, n_max = self._determine_n_to_use(out)
|
|
1391
|
+
|
|
1392
|
+
n_mode = self.request.n_mode
|
|
1393
|
+
n_avg = None
|
|
1394
|
+
if n_mode == 'average':
|
|
1395
|
+
n_avg = out['n'].mean()
|
|
1396
|
+
out['N'] = n_avg
|
|
1397
|
+
n_to_use = 'N'
|
|
1398
|
+
|
|
1399
|
+
out['center'] = _Xbar
|
|
1400
|
+
out[['lpl', 'upl']] = calculate_limits_vectorized(
|
|
1401
|
+
'Xbar',
|
|
1402
|
+
mean=out['center'],
|
|
1403
|
+
sd=_S,
|
|
1404
|
+
N=out[n_to_use],
|
|
1405
|
+
sigma_multiplier=self.request.n_sigma,
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
out = self._add_beyond_limits_flag(out, value_col='xbar')
|
|
1409
|
+
out = out.round(spec.round_to)
|
|
1410
|
+
|
|
1411
|
+
# Add stratum identifier
|
|
1412
|
+
out[stratify_col] = stratum
|
|
1413
|
+
|
|
1414
|
+
# Cast time column to string for categorical x-axis
|
|
1415
|
+
if groupby_cols:
|
|
1416
|
+
out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
|
|
1417
|
+
|
|
1418
|
+
all_xbar_frames.append(out)
|
|
1419
|
+
|
|
1420
|
+
_has_fixed_n = n_to_use == 'N'
|
|
1421
|
+
chart_statistics[stratum] = {
|
|
1422
|
+
'center': round(_Xbar, spec.round_to),
|
|
1423
|
+
'N': round(n_avg, spec.round_to) if n_avg is not None else (out['N'].iloc[0] if _has_fixed_n else None),
|
|
1424
|
+
'lpl': round(out['lpl'].iloc[0], spec.round_to) if _has_fixed_n else None,
|
|
1425
|
+
'upl': round(out['upl'].iloc[0], spec.round_to) if _has_fixed_n else None,
|
|
1426
|
+
**({'limits_vary': True} if not _has_fixed_n else {}),
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
if _return_intermediates:
|
|
1430
|
+
intermediates_per_stratum[stratum] = {
|
|
1431
|
+
'_S': _S,
|
|
1432
|
+
'n_to_use': n_to_use,
|
|
1433
|
+
'n_max': n_max,
|
|
1434
|
+
'out': out,
|
|
1435
|
+
'n_avg': n_avg,
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
if not all_xbar_frames:
|
|
1439
|
+
sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
|
|
1440
|
+
_raise_no_replication('Xbar', sds, insufficient_strata, len(strata))
|
|
1441
|
+
chart_out = pd.concat(all_xbar_frames, ignore_index=True)
|
|
1442
|
+
|
|
1443
|
+
# Publish only strata a caller can actually focus() into.
|
|
1444
|
+
published_strata = [s for s in strata if s not in set(insufficient_strata)]
|
|
1445
|
+
|
|
1446
|
+
# Select output columns
|
|
1447
|
+
group_col = groupby_cols[0] if groupby_cols else None
|
|
1448
|
+
cols_to_keep = [stratify_col, group_col, 'xbar', 'center', 'lpl', 'upl', 'beyond_limits']
|
|
1449
|
+
cols_to_keep = [c for c in cols_to_keep if c is not None and c in chart_out.columns]
|
|
1450
|
+
chart_out = chart_out[cols_to_keep]
|
|
1451
|
+
|
|
1452
|
+
result['Xbar'] = {
|
|
1453
|
+
'data': chart_out,
|
|
1454
|
+
'statistics': chart_statistics,
|
|
1455
|
+
'metadata': {
|
|
1456
|
+
'chart_type': 'Xbar',
|
|
1457
|
+
'value_col': 'xbar',
|
|
1458
|
+
'center_col': 'center',
|
|
1459
|
+
'stratified': True,
|
|
1460
|
+
'stratify_col': stratify_col,
|
|
1461
|
+
'stratify_by': list(stratify_by),
|
|
1462
|
+
'n_sigma': self.request.n_sigma,
|
|
1463
|
+
'n_mode': self.request.n_mode,
|
|
1464
|
+
'insufficient_strata': insufficient_strata or None,
|
|
1465
|
+
},
|
|
1466
|
+
'strata': published_strata,
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
if _return_intermediates:
|
|
1470
|
+
result['_intermediates'] = {
|
|
1471
|
+
'stratify_col': stratify_col,
|
|
1472
|
+
'stratify_by': stratify_by,
|
|
1473
|
+
# Published, not raw: the companion S path indexes per_stratum[stratum]
|
|
1474
|
+
# directly, and a skipped stratum is absent from it — passing the raw list
|
|
1475
|
+
# here raised KeyError on any partially-replicated study.
|
|
1476
|
+
'strata': published_strata,
|
|
1477
|
+
'insufficient_strata': insufficient_strata,
|
|
1478
|
+
'groupby_cols': groupby_cols,
|
|
1479
|
+
'per_stratum': intermediates_per_stratum,
|
|
1480
|
+
'is_stratified': True,
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
return result
|
|
1484
|
+
|
|
1485
|
+
def _calculate_s_stratified(
|
|
1486
|
+
self,
|
|
1487
|
+
value_col: str,
|
|
1488
|
+
groupby_cols: list[str],
|
|
1489
|
+
stratify_by: list[str],
|
|
1490
|
+
_precomputed: dict | None = None,
|
|
1491
|
+
) -> dict:
|
|
1492
|
+
"""
|
|
1493
|
+
Stratified S chart: one chart per factor combination, time on x-axis.
|
|
1494
|
+
|
|
1495
|
+
Parameters
|
|
1496
|
+
----------
|
|
1497
|
+
value_col : str
|
|
1498
|
+
Column to chart.
|
|
1499
|
+
groupby_cols : list[str]
|
|
1500
|
+
Columns to group by within each stratum.
|
|
1501
|
+
stratify_by : list[str]
|
|
1502
|
+
Columns defining strata.
|
|
1503
|
+
_precomputed : dict, optional
|
|
1504
|
+
Pre-computed intermediates from _calculate_xbar_stratified().
|
|
1505
|
+
|
|
1506
|
+
Returns
|
|
1507
|
+
-------
|
|
1508
|
+
dict
|
|
1509
|
+
Chart data with 'strata' key for stratified output.
|
|
1510
|
+
"""
|
|
1511
|
+
spec = self.spec
|
|
1512
|
+
|
|
1513
|
+
if _precomputed is not None:
|
|
1514
|
+
stratify_col = _precomputed['stratify_col']
|
|
1515
|
+
# Already filtered to the strata Xbar could compute; per_stratum has no entry
|
|
1516
|
+
# for the rest, so iterating the raw list would KeyError.
|
|
1517
|
+
strata = _precomputed['strata']
|
|
1518
|
+
per_stratum = _precomputed['per_stratum']
|
|
1519
|
+
insufficient_strata = list(_precomputed.get('insufficient_strata') or [])
|
|
1520
|
+
else:
|
|
1521
|
+
df = self._source_frame().copy()
|
|
1522
|
+
if len(stratify_by) == 1:
|
|
1523
|
+
stratify_col = stratify_by[0]
|
|
1524
|
+
else:
|
|
1525
|
+
df['_stratify_key'] = encode_rsg_series(df, stratify_by)
|
|
1526
|
+
stratify_col = '_stratify_key'
|
|
1527
|
+
strata = df[stratify_col].unique().tolist()
|
|
1528
|
+
per_stratum = None
|
|
1529
|
+
insufficient_strata = []
|
|
1530
|
+
|
|
1531
|
+
all_strata = list(strata) + [s for s in insufficient_strata if s not in strata]
|
|
1532
|
+
all_s_frames = []
|
|
1533
|
+
chart_statistics = {}
|
|
1534
|
+
|
|
1535
|
+
for stratum in strata:
|
|
1536
|
+
if per_stratum is not None:
|
|
1537
|
+
# Use precomputed data from Xbar stratified
|
|
1538
|
+
inter = per_stratum[stratum]
|
|
1539
|
+
out = inter['out'].copy()
|
|
1540
|
+
# Always use Xbar's _S (R2-based for residuals) for CL and limits
|
|
1541
|
+
_S = inter['_S']
|
|
1542
|
+
# Drop s_value — S chart plots R2 std (same basis as CL/limits)
|
|
1543
|
+
if 's_value' in out.columns:
|
|
1544
|
+
out = out.drop(columns=['s_value'])
|
|
1545
|
+
n_to_use = inter['n_to_use']
|
|
1546
|
+
n_max = inter['n_max']
|
|
1547
|
+
n_avg = inter['n_avg']
|
|
1548
|
+
else:
|
|
1549
|
+
sdf = df[df[stratify_col] == stratum].copy()
|
|
1550
|
+
_limits_col = self._resolve_limits_column(value_col, sdf)
|
|
1551
|
+
agg_dict = {
|
|
1552
|
+
's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
|
|
1553
|
+
'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
|
|
1554
|
+
}
|
|
1555
|
+
out = sdf.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
|
|
1556
|
+
mask = out['n'].eq(1)
|
|
1557
|
+
out = out[~mask]
|
|
1558
|
+
if out.shape[0] == 0:
|
|
1559
|
+
insufficient_strata.append(stratum)
|
|
1560
|
+
continue
|
|
1561
|
+
out['N'] = out['n'].max()
|
|
1562
|
+
# CL from R2-based std — S chart plots same basis
|
|
1563
|
+
_S = out['s'].mean()
|
|
1564
|
+
n_to_use, n_max = self._determine_n_to_use(out)
|
|
1565
|
+
n_avg = None
|
|
1566
|
+
if self.request.n_mode == 'average':
|
|
1567
|
+
n_avg = out['n'].mean()
|
|
1568
|
+
out['N'] = n_avg
|
|
1569
|
+
n_to_use = 'N'
|
|
1570
|
+
|
|
1571
|
+
# Cast time column to string for categorical x-axis
|
|
1572
|
+
if groupby_cols:
|
|
1573
|
+
out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
|
|
1574
|
+
|
|
1575
|
+
out['center'] = _S
|
|
1576
|
+
out[['lpl', 'upl']] = calculate_limits_vectorized(
|
|
1577
|
+
'S',
|
|
1578
|
+
sd=out['center'],
|
|
1579
|
+
N=out[n_to_use],
|
|
1580
|
+
sigma_multiplier=self.request.n_sigma,
|
|
1581
|
+
)
|
|
1582
|
+
|
|
1583
|
+
out = self._add_beyond_limits_flag(out, value_col='s')
|
|
1584
|
+
out = out.round(spec.round_to)
|
|
1585
|
+
out[stratify_col] = stratum
|
|
1586
|
+
|
|
1587
|
+
all_s_frames.append(out)
|
|
1588
|
+
|
|
1589
|
+
_has_fixed_n = n_to_use == 'N'
|
|
1590
|
+
chart_statistics[stratum] = {
|
|
1591
|
+
'center': round(_S, spec.round_to),
|
|
1592
|
+
'N': round(n_avg, spec.round_to) if n_avg is not None else (out['N'].iloc[0] if _has_fixed_n else None),
|
|
1593
|
+
'lpl': round(out['lpl'].iloc[0], spec.round_to) if _has_fixed_n else None,
|
|
1594
|
+
'upl': round(out['upl'].iloc[0], spec.round_to) if _has_fixed_n else None,
|
|
1595
|
+
**({'limits_vary': True} if not _has_fixed_n else {}),
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
if not all_s_frames:
|
|
1599
|
+
# Previously fell through to pd.concat([]), which surfaces as a raw pandas
|
|
1600
|
+
# "No objects to concatenate". The ungrouped S path and the stratified Xbar
|
|
1601
|
+
# path both already raised here; this one was the gap.
|
|
1602
|
+
sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
|
|
1603
|
+
_raise_no_replication('S', sds, insufficient_strata, len(all_strata))
|
|
1604
|
+
chart_out = pd.concat(all_s_frames, ignore_index=True)
|
|
1605
|
+
|
|
1606
|
+
published_strata = [s for s in all_strata if s not in set(insufficient_strata)]
|
|
1607
|
+
|
|
1608
|
+
group_col = groupby_cols[0] if groupby_cols else None
|
|
1609
|
+
cols_to_keep = [stratify_col, group_col, 's', 'center', 'lpl', 'upl', 'beyond_limits']
|
|
1610
|
+
cols_to_keep = [c for c in cols_to_keep if c is not None and c in chart_out.columns]
|
|
1611
|
+
chart_out = chart_out[cols_to_keep]
|
|
1612
|
+
|
|
1613
|
+
return {
|
|
1614
|
+
'S': {
|
|
1615
|
+
'data': chart_out,
|
|
1616
|
+
'statistics': chart_statistics,
|
|
1617
|
+
'metadata': {
|
|
1618
|
+
'chart_type': 'S',
|
|
1619
|
+
'value_col': 's',
|
|
1620
|
+
'center_col': 'center',
|
|
1621
|
+
'stratified': True,
|
|
1622
|
+
'stratify_col': stratify_col,
|
|
1623
|
+
'stratify_by': list(stratify_by),
|
|
1624
|
+
'n_sigma': self.request.n_sigma,
|
|
1625
|
+
'n_mode': self.request.n_mode,
|
|
1626
|
+
'insufficient_strata': insufficient_strata or None,
|
|
1627
|
+
},
|
|
1628
|
+
'strata': published_strata,
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
def _calculate_s( # noqa: C901
|
|
1633
|
+
self, value_col: str | None = None, _precomputed: dict | None = None
|
|
1634
|
+
) -> dict:
|
|
1635
|
+
"""
|
|
1636
|
+
Calculate S (standard deviation) chart statistics.
|
|
1637
|
+
|
|
1638
|
+
Parameters
|
|
1639
|
+
----------
|
|
1640
|
+
value_col : str, optional
|
|
1641
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
1642
|
+
Pass a residual column (R2, R5, etc.) for residual charts.
|
|
1643
|
+
_precomputed : dict, optional
|
|
1644
|
+
Pre-computed values from _calculate_xbar() to avoid redundant
|
|
1645
|
+
computation. Used internally by _calculate_xbar_s() for companion
|
|
1646
|
+
chart calculation. Keys: '_S', 'n_to_use', 'n_max', 'groupby_cols',
|
|
1647
|
+
'group_col', 'out'.
|
|
1648
|
+
|
|
1649
|
+
Returns
|
|
1650
|
+
-------
|
|
1651
|
+
dict
|
|
1652
|
+
Chart data: {'S': {'data': df, 'statistics': dict, 'metadata': dict}}
|
|
1653
|
+
|
|
1654
|
+
Logic moved from calculate_statistics_S()
|
|
1655
|
+
"""
|
|
1656
|
+
spec = self.spec
|
|
1657
|
+
if value_col is None:
|
|
1658
|
+
value_col = spec.response_var
|
|
1659
|
+
|
|
1660
|
+
# Use precomputed values if available (from _calculate_xbar_s)
|
|
1661
|
+
if _precomputed is not None:
|
|
1662
|
+
n_to_use = _precomputed['n_to_use']
|
|
1663
|
+
n_max = _precomputed['n_max']
|
|
1664
|
+
groupby_cols = _precomputed['groupby_cols']
|
|
1665
|
+
group_col = _precomputed['group_col']
|
|
1666
|
+
out = _precomputed['out'].copy()
|
|
1667
|
+
n_avg = _precomputed.get('n_avg')
|
|
1668
|
+
# Always use Xbar's _S (R2-based for residuals) for CL and limits
|
|
1669
|
+
_S = _precomputed['_S']
|
|
1670
|
+
# Drop s_value — S chart plots R2 std (same basis as CL/limits)
|
|
1671
|
+
if 's_value' in out.columns:
|
|
1672
|
+
out = out.drop(columns=['s_value'])
|
|
1673
|
+
else:
|
|
1674
|
+
# Independent calculation (existing logic)
|
|
1675
|
+
df = self._source_frame().copy()
|
|
1676
|
+
|
|
1677
|
+
# Resolve grouping based on `by` parameter
|
|
1678
|
+
groupby_cols, _, stratify_by = self._resolve_by_grouping(value_col)
|
|
1679
|
+
|
|
1680
|
+
# Stratified path: separate chart per factor combination
|
|
1681
|
+
if stratify_by:
|
|
1682
|
+
self._reject_calibration('stratified S')
|
|
1683
|
+
return self._calculate_s_stratified(
|
|
1684
|
+
value_col,
|
|
1685
|
+
groupby_cols,
|
|
1686
|
+
stratify_by,
|
|
1687
|
+
)
|
|
1688
|
+
|
|
1689
|
+
_limits_col = self._resolve_limits_column(value_col, df)
|
|
1690
|
+
|
|
1691
|
+
# Handle by=[] (collapse all) - single point chart
|
|
1692
|
+
if groupby_cols == []:
|
|
1693
|
+
_S = df[_limits_col].std()
|
|
1694
|
+
_N = len(df)
|
|
1695
|
+
out = pd.DataFrame({'group': ['All'], 's': [_S], 'n': [_N], 'N': [_N]})
|
|
1696
|
+
else:
|
|
1697
|
+
agg_dict = {
|
|
1698
|
+
's': pd.NamedAgg(column=_limits_col, aggfunc='std'),
|
|
1699
|
+
# Count on response_var (not value_col) to avoid NaN issues with residuals
|
|
1700
|
+
'n': pd.NamedAgg(column=spec.response_var, aggfunc='count'),
|
|
1701
|
+
}
|
|
1702
|
+
out = df.groupby(groupby_cols, as_index=False, observed=True).agg(**agg_dict)
|
|
1703
|
+
|
|
1704
|
+
# remove groups with a single observation
|
|
1705
|
+
mask = out['n'].eq(1)
|
|
1706
|
+
out = out[~mask]
|
|
1707
|
+
|
|
1708
|
+
# Handle case where no subgroups have >1 observation
|
|
1709
|
+
if out.shape[0] == 0:
|
|
1710
|
+
sds = self.ads._ads_result.sds if self.ads._ads_result else '?'
|
|
1711
|
+
_raise_no_replicated_subgroups('S', sds)
|
|
1712
|
+
|
|
1713
|
+
out['N'] = out['n'].max()
|
|
1714
|
+
|
|
1715
|
+
# CL from R2-based std — S chart plots same basis
|
|
1716
|
+
_S = out['s'].mean()
|
|
1717
|
+
|
|
1718
|
+
# Determine if subgroup sizes are constant or variable
|
|
1719
|
+
n_to_use, n_max = self._determine_n_to_use(out)
|
|
1720
|
+
|
|
1721
|
+
# Override n_to_use if n_mode="average"
|
|
1722
|
+
n_mode = self.request.n_mode
|
|
1723
|
+
n_avg = None
|
|
1724
|
+
if n_mode == 'average':
|
|
1725
|
+
n_avg = out['n'].mean()
|
|
1726
|
+
out['N'] = n_avg
|
|
1727
|
+
n_to_use = 'N'
|
|
1728
|
+
|
|
1729
|
+
# Determine group_col
|
|
1730
|
+
by = self.request.by
|
|
1731
|
+
if by is not None and len(by) > 0:
|
|
1732
|
+
if len(by) == 1:
|
|
1733
|
+
out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
|
|
1734
|
+
group_col = groupby_cols[0]
|
|
1735
|
+
else:
|
|
1736
|
+
out['subgroup'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1737
|
+
group_col = 'subgroup'
|
|
1738
|
+
elif len(groupby_cols) > 1:
|
|
1739
|
+
out['group'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1740
|
+
group_col = 'group'
|
|
1741
|
+
elif groupby_cols:
|
|
1742
|
+
group_col = groupby_cols[0]
|
|
1743
|
+
else:
|
|
1744
|
+
group_col = None
|
|
1745
|
+
|
|
1746
|
+
# Calculate S chart (common path for both precomputed and independent)
|
|
1747
|
+
out['groups'] = out['n'].count()
|
|
1748
|
+
n_mode = self.request.n_mode
|
|
1749
|
+
|
|
1750
|
+
if self.calibration is not None:
|
|
1751
|
+
# Standards-given S: center=c4(N)·sigma, B5(N)·sigma … B6(N)·sigma (per-row N).
|
|
1752
|
+
out[['center', 'lpl', 'upl']] = out.apply(
|
|
1753
|
+
lambda row: self._calibrated_row('S', N=row[n_to_use]),
|
|
1754
|
+
axis=1,
|
|
1755
|
+
)
|
|
1756
|
+
else:
|
|
1757
|
+
out['center'] = _S
|
|
1758
|
+
# Add limits columns
|
|
1759
|
+
out[['lpl', 'upl']] = calculate_limits_vectorized(
|
|
1760
|
+
'S',
|
|
1761
|
+
sd=out['center'],
|
|
1762
|
+
N=out[n_to_use],
|
|
1763
|
+
sigma_multiplier=self.request.n_sigma,
|
|
1764
|
+
)
|
|
1765
|
+
|
|
1766
|
+
# Detect beyond limits signals
|
|
1767
|
+
out = self._add_beyond_limits_flag(out, value_col='s')
|
|
1768
|
+
|
|
1769
|
+
# Apply grouping column transformation if needed and not already done
|
|
1770
|
+
if _precomputed is None:
|
|
1771
|
+
# Already handled in independent path above
|
|
1772
|
+
pass
|
|
1773
|
+
else:
|
|
1774
|
+
# For precomputed path, apply group_col transformation
|
|
1775
|
+
by = self.request.by
|
|
1776
|
+
if by is not None and len(by) > 0 and 'subgroup' not in out.columns:
|
|
1777
|
+
if len(by) == 1:
|
|
1778
|
+
out[groupby_cols[0]] = out[groupby_cols[0]].astype(str)
|
|
1779
|
+
group_col = groupby_cols[0]
|
|
1780
|
+
else:
|
|
1781
|
+
out['subgroup'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1782
|
+
group_col = 'subgroup'
|
|
1783
|
+
elif len(groupby_cols) > 1 and group_col == 'group' and 'group' not in out.columns:
|
|
1784
|
+
out['group'] = out[groupby_cols].astype(str).agg('_'.join, axis=1)
|
|
1785
|
+
|
|
1786
|
+
cols_to_keep = [group_col, 's', 'center', 'lpl', 'upl', 'beyond_limits']
|
|
1787
|
+
cols_to_keep = [c for c in cols_to_keep if c in out.columns]
|
|
1788
|
+
out = out[cols_to_keep]
|
|
1789
|
+
out = out.round(spec.round_to)
|
|
1790
|
+
|
|
1791
|
+
# Build statistics
|
|
1792
|
+
_S_stat = out['center'].iloc[0] if len(out) > 0 else None
|
|
1793
|
+
|
|
1794
|
+
statistics = {'center': round(_S_stat, spec.round_to) if _S_stat is not None else None}
|
|
1795
|
+
if n_to_use == 'N':
|
|
1796
|
+
statistics['N'] = round(n_avg, spec.round_to) if n_avg is not None else n_max
|
|
1797
|
+
statistics['upl'] = out['upl'].max()
|
|
1798
|
+
statistics['lpl'] = out['lpl'].max()
|
|
1799
|
+
else:
|
|
1800
|
+
statistics['N'] = None
|
|
1801
|
+
statistics['lpl'] = None
|
|
1802
|
+
statistics['upl'] = None
|
|
1803
|
+
statistics['limits_vary'] = True
|
|
1804
|
+
|
|
1805
|
+
return {
|
|
1806
|
+
'S': {
|
|
1807
|
+
'data': out,
|
|
1808
|
+
'statistics': statistics,
|
|
1809
|
+
'metadata': {
|
|
1810
|
+
'chart_type': 'S',
|
|
1811
|
+
'value_col': 's',
|
|
1812
|
+
'center_col': 'center',
|
|
1813
|
+
'n_sigma': self.request.n_sigma,
|
|
1814
|
+
'n_mode': n_mode,
|
|
1815
|
+
**({'n_avg': round(n_avg, spec.round_to)} if n_avg is not None else {}),
|
|
1816
|
+
},
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
def _calculate_xbar_s(self, value_col: str | None = None) -> dict[str, ChartPayload]:
|
|
1821
|
+
"""
|
|
1822
|
+
Calculate Xbar and S charts together (Bishop methodology).
|
|
1823
|
+
|
|
1824
|
+
This method ensures that when companion=True, both charts are calculated
|
|
1825
|
+
efficiently using shared intermediate values. The S chart uses the
|
|
1826
|
+
same aggregated data computed for Xbar, avoiding redundant calculation.
|
|
1827
|
+
|
|
1828
|
+
Parameters
|
|
1829
|
+
----------
|
|
1830
|
+
value_col : str, optional
|
|
1831
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
1832
|
+
|
|
1833
|
+
Returns
|
|
1834
|
+
-------
|
|
1835
|
+
dict
|
|
1836
|
+
Combined chart data: {'Xbar': {...}, 'S': {...}}
|
|
1837
|
+
"""
|
|
1838
|
+
# Calculate Xbar with intermediates for S reuse
|
|
1839
|
+
xbar_result = self._calculate_xbar(value_col, _return_intermediates=True)
|
|
1840
|
+
|
|
1841
|
+
# Extract intermediates and remove from result
|
|
1842
|
+
intermediates = xbar_result.pop('_intermediates')
|
|
1843
|
+
|
|
1844
|
+
# Stratified companion path
|
|
1845
|
+
if intermediates.get('is_stratified'):
|
|
1846
|
+
s_result = self._calculate_s_stratified(
|
|
1847
|
+
value_col,
|
|
1848
|
+
intermediates['groupby_cols'],
|
|
1849
|
+
intermediates['stratify_by'],
|
|
1850
|
+
_precomputed=intermediates,
|
|
1851
|
+
)
|
|
1852
|
+
return {**xbar_result, **s_result}
|
|
1853
|
+
|
|
1854
|
+
# Calculate S using precomputed values
|
|
1855
|
+
s_result = self._calculate_s(value_col, _precomputed=intermediates)
|
|
1856
|
+
|
|
1857
|
+
# Combine results
|
|
1858
|
+
return {**xbar_result, **s_result}
|
|
1859
|
+
|
|
1860
|
+
@staticmethod
|
|
1861
|
+
def _resolve_mr_source_column(value_col: str) -> str:
|
|
1862
|
+
"""Map recentered column (RCR*) to its non-recentered residual (R*).
|
|
1863
|
+
|
|
1864
|
+
Moving ranges must be computed from non-recentered residuals to avoid
|
|
1865
|
+
inflating mR with structural jumps between cells. Returns value_col
|
|
1866
|
+
unchanged if it is not a recentered column.
|
|
1867
|
+
"""
|
|
1868
|
+
if value_col.startswith('RCR'):
|
|
1869
|
+
return value_col[2:] # RCR3 -> R3, RCR6 -> R6, etc.
|
|
1870
|
+
return value_col
|
|
1871
|
+
|
|
1872
|
+
def _calculate_mr_chart(
|
|
1873
|
+
self,
|
|
1874
|
+
mr_spec: _MRChartSpec,
|
|
1875
|
+
value_col: str | None = None,
|
|
1876
|
+
_return_intermediates: bool = False,
|
|
1877
|
+
_precomputed: dict | None = None,
|
|
1878
|
+
) -> dict:
|
|
1879
|
+
"""
|
|
1880
|
+
Shared pipeline for MR-family charts (X and mR).
|
|
1881
|
+
|
|
1882
|
+
The X and mR charts share >85% of their calculation logic. The behavioral
|
|
1883
|
+
differences are encoded in ``mr_spec`` — no boolean flags needed.
|
|
1884
|
+
|
|
1885
|
+
Parameters
|
|
1886
|
+
----------
|
|
1887
|
+
mr_spec : _MRChartSpec
|
|
1888
|
+
Chart behavior specification (_XMR_SPEC or _R_SPEC).
|
|
1889
|
+
value_col : str, optional
|
|
1890
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
1891
|
+
_return_intermediates : bool, optional
|
|
1892
|
+
If True, includes '_intermediates' key for companion-mode reuse.
|
|
1893
|
+
_precomputed : dict, optional
|
|
1894
|
+
Pre-computed intermediates from a prior X call (companion mode).
|
|
1895
|
+
When provided, skips data preparation and uses these values directly.
|
|
1896
|
+
|
|
1897
|
+
Returns
|
|
1898
|
+
-------
|
|
1899
|
+
dict
|
|
1900
|
+
Chart data: {chart_type: {'data': df, 'statistics': dict, 'metadata': dict}}
|
|
1901
|
+
"""
|
|
1902
|
+
spec = self.spec
|
|
1903
|
+
if value_col is None:
|
|
1904
|
+
value_col = spec.response_var
|
|
1905
|
+
|
|
1906
|
+
logger.debug('In calculate statistics %s', mr_spec.chart_type)
|
|
1907
|
+
|
|
1908
|
+
# --- Determine stratification (shared logic) ---
|
|
1909
|
+
stratify_by, collapsed_factors = self._resolve_xmr_stratification()
|
|
1910
|
+
is_stratified = len(stratify_by) > 0
|
|
1911
|
+
|
|
1912
|
+
# --- Resolve plot column ---
|
|
1913
|
+
plot_col = value_col if mr_spec.plot_col == 'raw' else 'mr'
|
|
1914
|
+
|
|
1915
|
+
# --- Resolve mR source column ---
|
|
1916
|
+
# When recentered (value_col is RCR*), compute mR from the non-recentered
|
|
1917
|
+
# residual (R*) to avoid inflating moving ranges with structural jumps.
|
|
1918
|
+
mr_source_col = self._resolve_mr_source_column(value_col)
|
|
1919
|
+
|
|
1920
|
+
if is_stratified:
|
|
1921
|
+
self._reject_calibration('stratified X/mR')
|
|
1922
|
+
return self._calculate_mr_chart_stratified(
|
|
1923
|
+
mr_spec,
|
|
1924
|
+
value_col,
|
|
1925
|
+
plot_col,
|
|
1926
|
+
mr_source_col,
|
|
1927
|
+
stratify_by,
|
|
1928
|
+
collapsed_factors,
|
|
1929
|
+
_return_intermediates,
|
|
1930
|
+
_precomputed,
|
|
1931
|
+
phased=self.request.phased,
|
|
1932
|
+
)
|
|
1933
|
+
else:
|
|
1934
|
+
return self._calculate_mr_chart_ungrouped(
|
|
1935
|
+
mr_spec,
|
|
1936
|
+
value_col,
|
|
1937
|
+
plot_col,
|
|
1938
|
+
mr_source_col,
|
|
1939
|
+
collapsed_factors,
|
|
1940
|
+
_return_intermediates,
|
|
1941
|
+
_precomputed,
|
|
1942
|
+
phased=self.request.phased,
|
|
1943
|
+
)
|
|
1944
|
+
|
|
1945
|
+
def _calculate_mr_chart_from_precomputed(
|
|
1946
|
+
self,
|
|
1947
|
+
mr_spec: _MRChartSpec,
|
|
1948
|
+
plot_col: str,
|
|
1949
|
+
precomputed: dict,
|
|
1950
|
+
) -> dict:
|
|
1951
|
+
"""Build an MR-family chart reusing intermediates from a companion X call."""
|
|
1952
|
+
spec = self.spec
|
|
1953
|
+
out = precomputed['out'].copy()
|
|
1954
|
+
stratify_col = precomputed['stratify_col']
|
|
1955
|
+
stratify_by = precomputed['stratify_by']
|
|
1956
|
+
strata = precomputed['strata']
|
|
1957
|
+
xmr_lane_boundaries = precomputed['lane_boundaries']
|
|
1958
|
+
extra_cols = precomputed['extra_cols']
|
|
1959
|
+
|
|
1960
|
+
# --- Phased precomputed path ---
|
|
1961
|
+
if precomputed.get('phased'):
|
|
1962
|
+
phase_stats = precomputed['phase_stats']
|
|
1963
|
+
|
|
1964
|
+
# Recompute R-specific limits per phase within each stratum
|
|
1965
|
+
r_phase_stats = phase_stats[[stratify_col, '_phase_id', 'mR']].copy()
|
|
1966
|
+
r_lims = r_phase_stats.apply(
|
|
1967
|
+
lambda row: calculate_limits(
|
|
1968
|
+
mean=0,
|
|
1969
|
+
sd=0,
|
|
1970
|
+
N=0,
|
|
1971
|
+
mR=row['mR'],
|
|
1972
|
+
limits_type=mr_spec.limits_type,
|
|
1973
|
+
round_to=spec.round_to,
|
|
1974
|
+
),
|
|
1975
|
+
axis=1,
|
|
1976
|
+
)
|
|
1977
|
+
r_phase_stats = _join_limits(r_phase_stats, r_lims)
|
|
1978
|
+
r_phase_stats['center'] = r_phase_stats['mR']
|
|
1979
|
+
|
|
1980
|
+
# Replace X limits with mR limits
|
|
1981
|
+
out = out.drop(columns=['center', 'lpl', 'upl', 'beyond_limits'], errors='ignore')
|
|
1982
|
+
out = out.merge(
|
|
1983
|
+
r_phase_stats[[stratify_col, '_phase_id', 'center', 'lpl', 'upl']],
|
|
1984
|
+
on=[stratify_col, '_phase_id'],
|
|
1985
|
+
how='left',
|
|
1986
|
+
validate='many_to_one',
|
|
1987
|
+
)
|
|
1988
|
+
|
|
1989
|
+
# Re-detect beyond limits
|
|
1990
|
+
if plot_col not in out.columns:
|
|
1991
|
+
raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
|
|
1992
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
1993
|
+
|
|
1994
|
+
# Format output
|
|
1995
|
+
chart_out = self._build_output_columns(
|
|
1996
|
+
df=out,
|
|
1997
|
+
value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
chart_statistics = {
|
|
2001
|
+
stratum: {
|
|
2002
|
+
'N': None,
|
|
2003
|
+
'center': None,
|
|
2004
|
+
'lpl': None,
|
|
2005
|
+
'upl': None,
|
|
2006
|
+
'limits_vary': True,
|
|
2007
|
+
}
|
|
2008
|
+
for stratum in strata
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
# Phased R: no lane boundary offset (no rows dropped)
|
|
2012
|
+
return _build_stratified_mr_payload(
|
|
2013
|
+
mr_spec=mr_spec,
|
|
2014
|
+
chart_out=chart_out,
|
|
2015
|
+
chart_statistics=chart_statistics,
|
|
2016
|
+
plot_col=plot_col,
|
|
2017
|
+
stratify_col=stratify_col,
|
|
2018
|
+
stratify_by=stratify_by,
|
|
2019
|
+
strata=strata,
|
|
2020
|
+
lane_boundaries=xmr_lane_boundaries,
|
|
2021
|
+
phased=True,
|
|
2022
|
+
)
|
|
2023
|
+
|
|
2024
|
+
# --- Non-phased precomputed path ---
|
|
2025
|
+
grouped = precomputed['grouped'].copy()
|
|
2026
|
+
|
|
2027
|
+
# For mR chart: compute mR-specific limits from X's grouped data
|
|
2028
|
+
# grouped has columns: stratify_col, center (=mean), mR, lpl, upl (X limits)
|
|
2029
|
+
r_grouped = grouped.rename(columns={'center': 'xmr_center', 'mR': 'center'})
|
|
2030
|
+
|
|
2031
|
+
r_lims = r_grouped.apply(
|
|
2032
|
+
lambda row: calculate_limits(
|
|
2033
|
+
mean=0,
|
|
2034
|
+
sd=0,
|
|
2035
|
+
N=0,
|
|
2036
|
+
mR=row['center'],
|
|
2037
|
+
limits_type=mr_spec.limits_type,
|
|
2038
|
+
round_to=spec.round_to,
|
|
2039
|
+
),
|
|
2040
|
+
axis=1,
|
|
2041
|
+
)
|
|
2042
|
+
|
|
2043
|
+
r_grouped = _join_limits(r_grouped, r_lims, rsuffix='_r')
|
|
2044
|
+
|
|
2045
|
+
# Resolve lpl/upl column names (may have suffix if X lpl/upl exist)
|
|
2046
|
+
if 'lpl_r' in r_grouped.columns:
|
|
2047
|
+
r_grouped = r_grouped.rename(columns={'lpl_r': 'r_lpl', 'upl_r': 'r_upl'})
|
|
2048
|
+
r_lpl_col, r_upl_col = 'r_lpl', 'r_upl'
|
|
2049
|
+
else:
|
|
2050
|
+
r_lpl_col, r_upl_col = 'lpl', 'upl'
|
|
2051
|
+
|
|
2052
|
+
# Merge mR limits to the data (drop X-specific columns first)
|
|
2053
|
+
r_out = out.drop(columns=['center', 'lpl', 'upl', 'beyond_limits'], errors='ignore')
|
|
2054
|
+
r_merge_data = r_grouped[[stratify_col, 'center', r_lpl_col, r_upl_col]].rename(
|
|
2055
|
+
columns={r_lpl_col: 'lpl', r_upl_col: 'upl'}
|
|
2056
|
+
)
|
|
2057
|
+
r_out = r_out.merge(r_merge_data, on=stratify_col, how='left', validate='many_to_one')
|
|
2058
|
+
|
|
2059
|
+
# R chart drops first observation per stratum (NaN moving range)
|
|
2060
|
+
if mr_spec.drops_first_mr:
|
|
2061
|
+
r_out = r_out.dropna(subset=['mr'])
|
|
2062
|
+
|
|
2063
|
+
# Detect beyond limits
|
|
2064
|
+
r_out = self._add_beyond_limits_flag(r_out, value_col=plot_col)
|
|
2065
|
+
|
|
2066
|
+
# Build statistics
|
|
2067
|
+
chart_statistics = {}
|
|
2068
|
+
for stratum in strata:
|
|
2069
|
+
row = r_grouped[r_grouped[stratify_col] == stratum].iloc[0]
|
|
2070
|
+
chart_statistics[stratum] = {
|
|
2071
|
+
'N': _mr_family_subgroup_n(mr_spec.chart_type),
|
|
2072
|
+
'center': round(row['center'], spec.round_to),
|
|
2073
|
+
'lpl': round(row[r_lpl_col], spec.round_to),
|
|
2074
|
+
'upl': round(row[r_upl_col], spec.round_to),
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
# Format output
|
|
2078
|
+
chart_out = self._build_output_columns(
|
|
2079
|
+
df=r_out,
|
|
2080
|
+
value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2081
|
+
)
|
|
2082
|
+
|
|
2083
|
+
lane_boundaries = _shift_lane_boundaries(xmr_lane_boundaries, mr_spec.lane_boundary_offset)
|
|
2084
|
+
insufficient_strata, _ = _split_strata_by_sufficiency(r_out, stratify_col, strata)
|
|
2085
|
+
|
|
2086
|
+
return _build_stratified_mr_payload(
|
|
2087
|
+
mr_spec=mr_spec,
|
|
2088
|
+
chart_out=chart_out,
|
|
2089
|
+
chart_statistics=chart_statistics,
|
|
2090
|
+
plot_col=plot_col,
|
|
2091
|
+
stratify_col=stratify_col,
|
|
2092
|
+
stratify_by=stratify_by,
|
|
2093
|
+
strata=strata,
|
|
2094
|
+
lane_boundaries=lane_boundaries,
|
|
2095
|
+
insufficient_strata=insufficient_strata,
|
|
2096
|
+
)
|
|
2097
|
+
|
|
2098
|
+
def _calculate_mr_chart_stratified( # noqa: C901
|
|
2099
|
+
self,
|
|
2100
|
+
mr_spec: _MRChartSpec,
|
|
2101
|
+
value_col: str,
|
|
2102
|
+
plot_col: str,
|
|
2103
|
+
mr_source_col: str,
|
|
2104
|
+
stratify_by: list[str],
|
|
2105
|
+
collapsed_factors: list[str],
|
|
2106
|
+
_return_intermediates: bool,
|
|
2107
|
+
_precomputed: dict | None,
|
|
2108
|
+
phased: bool = False,
|
|
2109
|
+
) -> dict:
|
|
2110
|
+
"""Stratified path for _calculate_mr_chart."""
|
|
2111
|
+
spec = self.spec
|
|
2112
|
+
|
|
2113
|
+
if _precomputed is not None and _precomputed['is_stratified']:
|
|
2114
|
+
return self._calculate_mr_chart_from_precomputed(
|
|
2115
|
+
mr_spec,
|
|
2116
|
+
plot_col,
|
|
2117
|
+
_precomputed,
|
|
2118
|
+
)
|
|
2119
|
+
|
|
2120
|
+
# --- Independent calculation (no precomputed values) ---
|
|
2121
|
+
result = {}
|
|
2122
|
+
out = self._source_frame().copy()
|
|
2123
|
+
logger.debug('Dataframe has columns: %s', out.columns.to_list())
|
|
2124
|
+
|
|
2125
|
+
# Determine stratification column
|
|
2126
|
+
if stratify_by == [spec.rsg_var_name]:
|
|
2127
|
+
stratify_col = spec.rsg_var_name
|
|
2128
|
+
elif len(stratify_by) == 1:
|
|
2129
|
+
stratify_col = stratify_by[0]
|
|
2130
|
+
else:
|
|
2131
|
+
out['_stratify_key'] = encode_rsg_series(out, stratify_by)
|
|
2132
|
+
stratify_col = '_stratify_key'
|
|
2133
|
+
|
|
2134
|
+
# Canonical sort
|
|
2135
|
+
out = out.sort_values('sort_key', kind='stable')
|
|
2136
|
+
|
|
2137
|
+
# Strata list (needed for both phased and non-phased paths)
|
|
2138
|
+
strata = out[stratify_col].unique().tolist()
|
|
2139
|
+
|
|
2140
|
+
# === Phased limits path (per-phase limits within each stratum) ===
|
|
2141
|
+
if phased and collapsed_factors:
|
|
2142
|
+
# 1. Build phase key from collapsed factors only (vectorized)
|
|
2143
|
+
if len(collapsed_factors) == 1:
|
|
2144
|
+
out['_phase_key'] = out[collapsed_factors[0]]
|
|
2145
|
+
else:
|
|
2146
|
+
out['_phase_key'] = list(map(tuple, out[collapsed_factors].to_numpy()))
|
|
2147
|
+
|
|
2148
|
+
# 2. Assign within-stratum phase IDs via transitions of _phase_key
|
|
2149
|
+
out['_phase_id'] = (
|
|
2150
|
+
out.groupby(stratify_col, sort=False)['_phase_key']
|
|
2151
|
+
.transform(lambda s: (s != s.shift()).cumsum())
|
|
2152
|
+
.astype('int64')
|
|
2153
|
+
)
|
|
2154
|
+
|
|
2155
|
+
# 3. Per-phase mR (reset at phase boundaries within each stratum)
|
|
2156
|
+
out['mr'] = out.groupby([stratify_col, '_phase_id'], sort=False)[mr_source_col].diff().abs()
|
|
2157
|
+
|
|
2158
|
+
# 4. Per-phase aggregation
|
|
2159
|
+
phase_stats = (
|
|
2160
|
+
out.groupby([stratify_col, '_phase_id'], sort=False)
|
|
2161
|
+
.agg(_n=(value_col, 'size'), _mean=(value_col, 'mean'), mR=('mr', 'mean'))
|
|
2162
|
+
.reset_index()
|
|
2163
|
+
)
|
|
2164
|
+
phase_stats['mR'] = phase_stats['mR'].fillna(0.0)
|
|
2165
|
+
|
|
2166
|
+
# 5. Compute per-phase limits
|
|
2167
|
+
if mr_spec.center_source == 'mean':
|
|
2168
|
+
phase_stats['center'] = phase_stats['_mean']
|
|
2169
|
+
phase_lims = phase_stats.apply(
|
|
2170
|
+
lambda row: calculate_limits(
|
|
2171
|
+
mean=row['_mean'],
|
|
2172
|
+
sd=0,
|
|
2173
|
+
N=0,
|
|
2174
|
+
mR=row['mR'],
|
|
2175
|
+
limits_type=mr_spec.limits_type,
|
|
2176
|
+
round_to=spec.round_to,
|
|
2177
|
+
),
|
|
2178
|
+
axis=1,
|
|
2179
|
+
)
|
|
2180
|
+
else: # center_source == 'mR' (R chart)
|
|
2181
|
+
phase_stats['center'] = phase_stats['mR']
|
|
2182
|
+
phase_lims = phase_stats.apply(
|
|
2183
|
+
lambda row: calculate_limits(
|
|
2184
|
+
mean=0,
|
|
2185
|
+
sd=0,
|
|
2186
|
+
N=0,
|
|
2187
|
+
mR=row['mR'],
|
|
2188
|
+
limits_type=mr_spec.limits_type,
|
|
2189
|
+
round_to=spec.round_to,
|
|
2190
|
+
),
|
|
2191
|
+
axis=1,
|
|
2192
|
+
)
|
|
2193
|
+
phase_stats = _join_limits(phase_stats, phase_lims)
|
|
2194
|
+
|
|
2195
|
+
# 6. Merge per-phase limits back to row-level data
|
|
2196
|
+
out = out.merge(
|
|
2197
|
+
phase_stats[[stratify_col, '_phase_id', 'center', 'mR', 'lpl', 'upl']],
|
|
2198
|
+
on=[stratify_col, '_phase_id'],
|
|
2199
|
+
how='left',
|
|
2200
|
+
validate='many_to_one',
|
|
2201
|
+
)
|
|
2202
|
+
|
|
2203
|
+
# 7. Phased path: NEVER drop rows. NaN mR at phase boundaries preserved.
|
|
2204
|
+
|
|
2205
|
+
# 8. Signal detection (per-row, uses row-level lpl/upl)
|
|
2206
|
+
if plot_col not in out.columns:
|
|
2207
|
+
raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
|
|
2208
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2209
|
+
|
|
2210
|
+
# 9. Lane boundaries per stratum
|
|
2211
|
+
lane_boundaries = {}
|
|
2212
|
+
if collapsed_factors:
|
|
2213
|
+
for stratum in strata:
|
|
2214
|
+
stratum_data = out[out[stratify_col] == stratum].reset_index(drop=True)
|
|
2215
|
+
boundaries = self._calculate_lane_boundaries(stratum_data, collapsed_factors)
|
|
2216
|
+
if boundaries:
|
|
2217
|
+
lane_boundaries[stratum] = boundaries
|
|
2218
|
+
|
|
2219
|
+
# 10. Statistics per stratum: limits vary per phase
|
|
2220
|
+
chart_statistics = {
|
|
2221
|
+
stratum: {
|
|
2222
|
+
'N': None,
|
|
2223
|
+
'center': None,
|
|
2224
|
+
'lpl': None,
|
|
2225
|
+
'upl': None,
|
|
2226
|
+
'limits_vary': True,
|
|
2227
|
+
}
|
|
2228
|
+
for stratum in strata
|
|
2229
|
+
}
|
|
2230
|
+
|
|
2231
|
+
# 11. Track single-point phases
|
|
2232
|
+
n_single_point = int((phase_stats['_n'] < 2).sum())
|
|
2233
|
+
|
|
2234
|
+
# Format output
|
|
2235
|
+
extra_cols = [c for c in stratify_by if c != spec.rsg_var_name and c != spec.time_var]
|
|
2236
|
+
chart_out = self._build_output_columns(
|
|
2237
|
+
df=out,
|
|
2238
|
+
value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
result.update(_build_stratified_mr_payload(
|
|
2242
|
+
mr_spec=mr_spec,
|
|
2243
|
+
chart_out=chart_out,
|
|
2244
|
+
chart_statistics=chart_statistics,
|
|
2245
|
+
plot_col=plot_col,
|
|
2246
|
+
stratify_col=stratify_col,
|
|
2247
|
+
stratify_by=stratify_by,
|
|
2248
|
+
strata=strata,
|
|
2249
|
+
lane_boundaries=lane_boundaries,
|
|
2250
|
+
phased=True,
|
|
2251
|
+
single_point_phases=n_single_point,
|
|
2252
|
+
))
|
|
2253
|
+
|
|
2254
|
+
if _return_intermediates:
|
|
2255
|
+
result['_intermediates'] = {
|
|
2256
|
+
'out': out,
|
|
2257
|
+
'phase_stats': phase_stats,
|
|
2258
|
+
'stratify_col': stratify_col,
|
|
2259
|
+
'stratify_by': stratify_by,
|
|
2260
|
+
'strata': strata,
|
|
2261
|
+
'lane_boundaries': lane_boundaries,
|
|
2262
|
+
'collapsed_factors': collapsed_factors,
|
|
2263
|
+
'extra_cols': extra_cols,
|
|
2264
|
+
'is_stratified': True,
|
|
2265
|
+
'phased': True,
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
return result
|
|
2269
|
+
|
|
2270
|
+
# === Global limits path (unchanged) ===
|
|
2271
|
+
|
|
2272
|
+
# Moving range per stratum
|
|
2273
|
+
out['mr'] = out.groupby(stratify_col, sort=False, observed=True)[mr_source_col].diff().abs()
|
|
2274
|
+
|
|
2275
|
+
# Aggregate per stratum
|
|
2276
|
+
agg = {}
|
|
2277
|
+
if value_col in out.columns:
|
|
2278
|
+
agg['mean'] = (value_col, 'mean')
|
|
2279
|
+
if 'mr' in out.columns:
|
|
2280
|
+
agg['mR'] = ('mr', 'mean')
|
|
2281
|
+
|
|
2282
|
+
if not agg:
|
|
2283
|
+
raise RuntimeError(
|
|
2284
|
+
f'{mr_spec.chart_type} aggregation spec is empty. '
|
|
2285
|
+
f'Have columns: {out.columns.tolist()}, value_col={value_col!r}'
|
|
2286
|
+
)
|
|
2287
|
+
|
|
2288
|
+
grouped = out.groupby(stratify_col, sort=False, observed=True).agg(**agg).reset_index()
|
|
2289
|
+
|
|
2290
|
+
# Compute limits per group
|
|
2291
|
+
# For X: center=mean, limits based on mean ± E2*mR
|
|
2292
|
+
# For mR: center=mR, limits based on 0 to D4*mR
|
|
2293
|
+
if mr_spec.center_source == 'mean':
|
|
2294
|
+
lims = grouped.apply(
|
|
2295
|
+
lambda row: calculate_limits(
|
|
2296
|
+
mean=row['mean'],
|
|
2297
|
+
sd=0,
|
|
2298
|
+
N=0,
|
|
2299
|
+
mR=(row['mR'] if pd.notna(row['mR']) else 0.0),
|
|
2300
|
+
limits_type=mr_spec.limits_type,
|
|
2301
|
+
round_to=spec.round_to,
|
|
2302
|
+
),
|
|
2303
|
+
axis=1,
|
|
2304
|
+
)
|
|
2305
|
+
else: # center_source == 'mR'
|
|
2306
|
+
lims = grouped.apply(
|
|
2307
|
+
lambda row: calculate_limits(
|
|
2308
|
+
mean=0,
|
|
2309
|
+
sd=0,
|
|
2310
|
+
N=0,
|
|
2311
|
+
mR=row['mR'],
|
|
2312
|
+
limits_type=mr_spec.limits_type,
|
|
2313
|
+
round_to=spec.round_to,
|
|
2314
|
+
),
|
|
2315
|
+
axis=1,
|
|
2316
|
+
)
|
|
2317
|
+
|
|
2318
|
+
# Normalize/join lpl/upl
|
|
2319
|
+
grouped = _join_limits(grouped, lims)
|
|
2320
|
+
|
|
2321
|
+
# Set center column based on chart type
|
|
2322
|
+
if mr_spec.center_source == 'mean':
|
|
2323
|
+
grouped = grouped.rename(columns={'mean': 'center'})
|
|
2324
|
+
else:
|
|
2325
|
+
grouped['center'] = grouped['mR']
|
|
2326
|
+
|
|
2327
|
+
# Merge limits back to row-level data
|
|
2328
|
+
out = out.merge(
|
|
2329
|
+
grouped[[stratify_col, 'center', 'mR', 'lpl', 'upl']],
|
|
2330
|
+
on=stratify_col,
|
|
2331
|
+
how='left',
|
|
2332
|
+
validate='many_to_one',
|
|
2333
|
+
)
|
|
2334
|
+
|
|
2335
|
+
# R chart: drop first observation per stratum (NaN moving range)
|
|
2336
|
+
if mr_spec.drops_first_mr:
|
|
2337
|
+
out = out.dropna(subset=['mr'])
|
|
2338
|
+
|
|
2339
|
+
# Detect beyond limits
|
|
2340
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2341
|
+
|
|
2342
|
+
# Use strata from grouped for non-phased path (preserves original order)
|
|
2343
|
+
strata = grouped[stratify_col].tolist()
|
|
2344
|
+
|
|
2345
|
+
# Lane boundaries per stratum
|
|
2346
|
+
lane_boundaries = {}
|
|
2347
|
+
if collapsed_factors:
|
|
2348
|
+
for stratum in strata:
|
|
2349
|
+
stratum_data = out[out[stratify_col] == stratum].reset_index(drop=True)
|
|
2350
|
+
boundaries = self._calculate_lane_boundaries(stratum_data, collapsed_factors)
|
|
2351
|
+
if boundaries:
|
|
2352
|
+
lane_boundaries[stratum] = boundaries
|
|
2353
|
+
|
|
2354
|
+
# Build statistics nested by stratum
|
|
2355
|
+
chart_statistics = {}
|
|
2356
|
+
for stratum in strata:
|
|
2357
|
+
row = grouped[grouped[stratify_col] == stratum].iloc[0]
|
|
2358
|
+
chart_statistics[stratum] = {
|
|
2359
|
+
'N': _mr_family_subgroup_n(mr_spec.chart_type),
|
|
2360
|
+
'center': round(row['center'], spec.round_to),
|
|
2361
|
+
'lpl': round(row['lpl'], spec.round_to),
|
|
2362
|
+
'upl': round(row['upl'], spec.round_to),
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
# Format output
|
|
2366
|
+
extra_cols = [c for c in stratify_by if c != spec.rsg_var_name and c != spec.time_var]
|
|
2367
|
+
chart_out = self._build_output_columns(
|
|
2368
|
+
df=out,
|
|
2369
|
+
value_cols=extra_cols + [plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2370
|
+
)
|
|
2371
|
+
|
|
2372
|
+
insufficient_strata, _ = _split_strata_by_sufficiency(out, stratify_col, strata)
|
|
2373
|
+
|
|
2374
|
+
result.update(_build_stratified_mr_payload(
|
|
2375
|
+
mr_spec=mr_spec,
|
|
2376
|
+
chart_out=chart_out,
|
|
2377
|
+
chart_statistics=chart_statistics,
|
|
2378
|
+
plot_col=plot_col,
|
|
2379
|
+
stratify_col=stratify_col,
|
|
2380
|
+
stratify_by=stratify_by,
|
|
2381
|
+
strata=strata,
|
|
2382
|
+
lane_boundaries=lane_boundaries,
|
|
2383
|
+
insufficient_strata=insufficient_strata,
|
|
2384
|
+
))
|
|
2385
|
+
|
|
2386
|
+
# Optionally include intermediates for companion calculation
|
|
2387
|
+
if _return_intermediates:
|
|
2388
|
+
result['_intermediates'] = {
|
|
2389
|
+
'out': out,
|
|
2390
|
+
'grouped': grouped,
|
|
2391
|
+
'stratify_col': stratify_col,
|
|
2392
|
+
'stratify_by': stratify_by,
|
|
2393
|
+
'strata': strata,
|
|
2394
|
+
'lane_boundaries': lane_boundaries,
|
|
2395
|
+
'collapsed_factors': collapsed_factors,
|
|
2396
|
+
'extra_cols': extra_cols,
|
|
2397
|
+
'is_stratified': True,
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
return result
|
|
2401
|
+
|
|
2402
|
+
def _calculate_mr_chart_ungrouped( # noqa: C901
|
|
2403
|
+
self,
|
|
2404
|
+
mr_spec: _MRChartSpec,
|
|
2405
|
+
value_col: str,
|
|
2406
|
+
plot_col: str,
|
|
2407
|
+
mr_source_col: str,
|
|
2408
|
+
collapsed_factors: list[str],
|
|
2409
|
+
_return_intermediates: bool,
|
|
2410
|
+
_precomputed: dict | None,
|
|
2411
|
+
phased: bool = False,
|
|
2412
|
+
) -> dict:
|
|
2413
|
+
"""Ungrouped (single-stream) path for _calculate_mr_chart."""
|
|
2414
|
+
spec = self.spec
|
|
2415
|
+
result = {}
|
|
2416
|
+
|
|
2417
|
+
if _precomputed is not None and not _precomputed['is_stratified']:
|
|
2418
|
+
# --- Phased precomputed path (for companion R chart) ---
|
|
2419
|
+
if _precomputed.get('phased'):
|
|
2420
|
+
out = _precomputed['out'].copy()
|
|
2421
|
+
phase_stats = _precomputed['phase_stats']
|
|
2422
|
+
xmr_lane_boundaries = _precomputed['lane_boundaries']
|
|
2423
|
+
|
|
2424
|
+
# Recompute R-specific limits per phase
|
|
2425
|
+
r_phase_stats = phase_stats[['_phase_id', 'mR']].copy()
|
|
2426
|
+
r_lims = r_phase_stats.apply(
|
|
2427
|
+
lambda row: calculate_limits(
|
|
2428
|
+
mean=0,
|
|
2429
|
+
sd=0,
|
|
2430
|
+
N=0,
|
|
2431
|
+
mR=row['mR'],
|
|
2432
|
+
limits_type=mr_spec.limits_type,
|
|
2433
|
+
round_to=spec.round_to,
|
|
2434
|
+
),
|
|
2435
|
+
axis=1,
|
|
2436
|
+
)
|
|
2437
|
+
r_phase_stats = _join_limits(r_phase_stats, r_lims)
|
|
2438
|
+
r_phase_stats['center'] = r_phase_stats['mR']
|
|
2439
|
+
|
|
2440
|
+
# Replace X limits with mR limits
|
|
2441
|
+
out = out.drop(columns=['center', 'lpl', 'upl'], errors='ignore')
|
|
2442
|
+
out = out.merge(
|
|
2443
|
+
r_phase_stats[['_phase_id', 'center', 'lpl', 'upl']],
|
|
2444
|
+
on='_phase_id',
|
|
2445
|
+
how='left',
|
|
2446
|
+
validate='many_to_one',
|
|
2447
|
+
)
|
|
2448
|
+
|
|
2449
|
+
# Re-detect beyond limits
|
|
2450
|
+
out = out.drop(columns=['beyond_limits'], errors='ignore')
|
|
2451
|
+
if plot_col not in out.columns:
|
|
2452
|
+
raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
|
|
2453
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2454
|
+
|
|
2455
|
+
chart_out = self._build_output_columns(
|
|
2456
|
+
df=out,
|
|
2457
|
+
value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2458
|
+
)
|
|
2459
|
+
|
|
2460
|
+
chart_statistics = {
|
|
2461
|
+
'N': None,
|
|
2462
|
+
'center': None,
|
|
2463
|
+
'lpl': None,
|
|
2464
|
+
'upl': None,
|
|
2465
|
+
'limits_vary': True,
|
|
2466
|
+
}
|
|
2467
|
+
|
|
2468
|
+
# Phased R: no lane boundary offset needed (no rows dropped)
|
|
2469
|
+
result.update(_build_ungrouped_mr_payload(
|
|
2470
|
+
mr_spec=mr_spec,
|
|
2471
|
+
chart_out=chart_out,
|
|
2472
|
+
chart_statistics=chart_statistics,
|
|
2473
|
+
plot_col=plot_col,
|
|
2474
|
+
lane_boundaries=xmr_lane_boundaries,
|
|
2475
|
+
phased=True,
|
|
2476
|
+
))
|
|
2477
|
+
return result
|
|
2478
|
+
|
|
2479
|
+
# --- Existing non-phased precomputed path ---
|
|
2480
|
+
# Reuse intermediates from companion X call
|
|
2481
|
+
out = _precomputed['out'].copy()
|
|
2482
|
+
mR = _precomputed['mR']
|
|
2483
|
+
xmr_lane_boundaries = _precomputed['lane_boundaries']
|
|
2484
|
+
|
|
2485
|
+
# R chart: drop first observation (NaN moving range)
|
|
2486
|
+
if mr_spec.drops_first_mr:
|
|
2487
|
+
out = out.dropna(subset=['mr'])
|
|
2488
|
+
|
|
2489
|
+
# Compute R-specific limits
|
|
2490
|
+
if self.calibration is not None:
|
|
2491
|
+
center_r, r_lims = self._calibrated_scalar(mr_spec.limits_type)
|
|
2492
|
+
else:
|
|
2493
|
+
center_r = mR
|
|
2494
|
+
r_lims = calculate_limits(
|
|
2495
|
+
mean=0,
|
|
2496
|
+
sd=0,
|
|
2497
|
+
N=0,
|
|
2498
|
+
mR=mR,
|
|
2499
|
+
limits_type=mr_spec.limits_type,
|
|
2500
|
+
round_to=spec.round_to,
|
|
2501
|
+
)
|
|
2502
|
+
out['center'] = center_r
|
|
2503
|
+
out['lpl'] = r_lims['lpl']
|
|
2504
|
+
out['upl'] = r_lims['upl']
|
|
2505
|
+
|
|
2506
|
+
# Re-detect beyond limits for R chart
|
|
2507
|
+
out = out.drop(columns=['beyond_limits'], errors='ignore')
|
|
2508
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2509
|
+
|
|
2510
|
+
# Format output
|
|
2511
|
+
chart_out = self._build_output_columns(
|
|
2512
|
+
df=out,
|
|
2513
|
+
value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2514
|
+
)
|
|
2515
|
+
|
|
2516
|
+
chart_statistics = {
|
|
2517
|
+
'N': _mr_family_subgroup_n(mr_spec.chart_type),
|
|
2518
|
+
'center': round(center_r, spec.round_to),
|
|
2519
|
+
'lpl': round(r_lims['lpl'], spec.round_to),
|
|
2520
|
+
'upl': round(r_lims['upl'], spec.round_to),
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
r_lane_boundaries = _shift_lane_boundaries(
|
|
2524
|
+
xmr_lane_boundaries, mr_spec.lane_boundary_offset
|
|
2525
|
+
)
|
|
2526
|
+
|
|
2527
|
+
result.update(_build_ungrouped_mr_payload(
|
|
2528
|
+
mr_spec=mr_spec,
|
|
2529
|
+
chart_out=chart_out,
|
|
2530
|
+
chart_statistics=chart_statistics,
|
|
2531
|
+
plot_col=plot_col,
|
|
2532
|
+
lane_boundaries=r_lane_boundaries,
|
|
2533
|
+
))
|
|
2534
|
+
return result
|
|
2535
|
+
|
|
2536
|
+
# --- Independent calculation ---
|
|
2537
|
+
out = self._source_frame().copy()
|
|
2538
|
+
out = out.sort_values('sort_key', kind='stable')
|
|
2539
|
+
out = out.reset_index(drop=True)
|
|
2540
|
+
|
|
2541
|
+
# Lane boundaries before any row drops
|
|
2542
|
+
lane_boundaries = None
|
|
2543
|
+
if collapsed_factors:
|
|
2544
|
+
lane_boundaries = self._calculate_lane_boundaries(out, collapsed_factors)
|
|
2545
|
+
|
|
2546
|
+
# === Phased limits path ===
|
|
2547
|
+
if phased and collapsed_factors:
|
|
2548
|
+
self._reject_calibration('phased X/mR')
|
|
2549
|
+
# 1. Assign phase_id: contiguous runs of the same rsg_key
|
|
2550
|
+
out['_phase_id'] = (out['rsg_key'] != out['rsg_key'].shift()).cumsum().astype('int64')
|
|
2551
|
+
|
|
2552
|
+
# 2. Per-phase moving range (reset at boundaries)
|
|
2553
|
+
out['mr'] = out.groupby('_phase_id', sort=False)[mr_source_col].diff().abs()
|
|
2554
|
+
|
|
2555
|
+
# 3. Per-phase aggregation
|
|
2556
|
+
phase_stats = (
|
|
2557
|
+
out.groupby('_phase_id', sort=False)
|
|
2558
|
+
.agg(
|
|
2559
|
+
_n=(value_col, 'size'),
|
|
2560
|
+
_mean=(value_col, 'mean'),
|
|
2561
|
+
mR=('mr', 'mean'),
|
|
2562
|
+
)
|
|
2563
|
+
.reset_index()
|
|
2564
|
+
)
|
|
2565
|
+
|
|
2566
|
+
# 4. Handle single-point phases (mR=NaN -> use 0)
|
|
2567
|
+
phase_stats['mR'] = phase_stats['mR'].fillna(0.0)
|
|
2568
|
+
|
|
2569
|
+
# 5. Compute per-phase limits
|
|
2570
|
+
if mr_spec.center_source == 'mean':
|
|
2571
|
+
phase_stats['center'] = phase_stats['_mean']
|
|
2572
|
+
phase_lims = phase_stats.apply(
|
|
2573
|
+
lambda row: calculate_limits(
|
|
2574
|
+
mean=row['_mean'],
|
|
2575
|
+
sd=0,
|
|
2576
|
+
N=0,
|
|
2577
|
+
mR=row['mR'],
|
|
2578
|
+
limits_type=mr_spec.limits_type,
|
|
2579
|
+
round_to=spec.round_to,
|
|
2580
|
+
),
|
|
2581
|
+
axis=1,
|
|
2582
|
+
)
|
|
2583
|
+
else: # center_source == 'mR' (R chart)
|
|
2584
|
+
phase_stats['center'] = phase_stats['mR']
|
|
2585
|
+
phase_lims = phase_stats.apply(
|
|
2586
|
+
lambda row: calculate_limits(
|
|
2587
|
+
mean=0,
|
|
2588
|
+
sd=0,
|
|
2589
|
+
N=0,
|
|
2590
|
+
mR=row['mR'],
|
|
2591
|
+
limits_type=mr_spec.limits_type,
|
|
2592
|
+
round_to=spec.round_to,
|
|
2593
|
+
),
|
|
2594
|
+
axis=1,
|
|
2595
|
+
)
|
|
2596
|
+
phase_stats = _join_limits(phase_stats, phase_lims)
|
|
2597
|
+
|
|
2598
|
+
# 6. Merge per-phase limits back to row-level data
|
|
2599
|
+
out = out.merge(
|
|
2600
|
+
phase_stats[['_phase_id', 'center', 'mR', 'lpl', 'upl']],
|
|
2601
|
+
on='_phase_id',
|
|
2602
|
+
how='left',
|
|
2603
|
+
validate='many_to_one',
|
|
2604
|
+
)
|
|
2605
|
+
|
|
2606
|
+
# 7. Phased path: NEVER drop rows. NaN mR values at phase boundaries
|
|
2607
|
+
# stay in the DataFrame. Plotly skips NaN naturally.
|
|
2608
|
+
|
|
2609
|
+
# 8. Signal detection (per-row, uses row-level lpl/upl)
|
|
2610
|
+
if plot_col not in out.columns:
|
|
2611
|
+
raise RuntimeError(f"plot_col {plot_col!r} not in DataFrame")
|
|
2612
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2613
|
+
|
|
2614
|
+
# 9. Format output
|
|
2615
|
+
chart_out = self._build_output_columns(
|
|
2616
|
+
df=out,
|
|
2617
|
+
value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2618
|
+
)
|
|
2619
|
+
|
|
2620
|
+
# 10. Statistics: limits_vary=True signals stepped rendering
|
|
2621
|
+
chart_statistics = {
|
|
2622
|
+
'N': None,
|
|
2623
|
+
'center': None,
|
|
2624
|
+
'lpl': None,
|
|
2625
|
+
'upl': None,
|
|
2626
|
+
'limits_vary': True,
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
# 11. Track single-point phases for visibility
|
|
2630
|
+
n_single_point = int((phase_stats['_n'] < 2).sum())
|
|
2631
|
+
|
|
2632
|
+
result.update(_build_ungrouped_mr_payload(
|
|
2633
|
+
mr_spec=mr_spec,
|
|
2634
|
+
chart_out=chart_out,
|
|
2635
|
+
chart_statistics=chart_statistics,
|
|
2636
|
+
plot_col=plot_col,
|
|
2637
|
+
lane_boundaries=lane_boundaries,
|
|
2638
|
+
phased=True,
|
|
2639
|
+
single_point_phases=n_single_point,
|
|
2640
|
+
))
|
|
2641
|
+
|
|
2642
|
+
if _return_intermediates:
|
|
2643
|
+
result['_intermediates'] = {
|
|
2644
|
+
'out': out,
|
|
2645
|
+
'mR': None,
|
|
2646
|
+
'lane_boundaries': lane_boundaries,
|
|
2647
|
+
'is_stratified': False,
|
|
2648
|
+
'phased': True,
|
|
2649
|
+
'phase_stats': phase_stats,
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
return result
|
|
2653
|
+
|
|
2654
|
+
# === Global limits path (unchanged) ===
|
|
2655
|
+
# Moving range
|
|
2656
|
+
out['mr'] = out[mr_source_col].diff().abs()
|
|
2657
|
+
mR = out['mr'].mean()
|
|
2658
|
+
mean_ = out[value_col].mean()
|
|
2659
|
+
|
|
2660
|
+
# R chart: drop first observation
|
|
2661
|
+
if mr_spec.drops_first_mr:
|
|
2662
|
+
out = out.dropna(subset=['mr'])
|
|
2663
|
+
|
|
2664
|
+
# Compute limits
|
|
2665
|
+
if self.calibration is not None:
|
|
2666
|
+
# Standards-given: X -> mean ± 3·sigma; mR -> 0 … D4·d2·sigma.
|
|
2667
|
+
center_val, lims = self._calibrated_scalar(mr_spec.limits_type)
|
|
2668
|
+
elif mr_spec.center_source == 'mean':
|
|
2669
|
+
center_val = mean_
|
|
2670
|
+
lims = calculate_limits(
|
|
2671
|
+
mean=mean_,
|
|
2672
|
+
sd=0,
|
|
2673
|
+
N=0,
|
|
2674
|
+
mR=mR,
|
|
2675
|
+
limits_type=mr_spec.limits_type,
|
|
2676
|
+
round_to=spec.round_to,
|
|
2677
|
+
)
|
|
2678
|
+
else:
|
|
2679
|
+
center_val = mR
|
|
2680
|
+
lims = calculate_limits(
|
|
2681
|
+
mean=0,
|
|
2682
|
+
sd=0,
|
|
2683
|
+
N=0,
|
|
2684
|
+
mR=mR,
|
|
2685
|
+
limits_type=mr_spec.limits_type,
|
|
2686
|
+
round_to=spec.round_to,
|
|
2687
|
+
)
|
|
2688
|
+
|
|
2689
|
+
out['center'] = center_val
|
|
2690
|
+
out['mR'] = mR
|
|
2691
|
+
out['lpl'] = lims['lpl']
|
|
2692
|
+
out['upl'] = lims['upl']
|
|
2693
|
+
|
|
2694
|
+
# Detect beyond limits
|
|
2695
|
+
out = self._add_beyond_limits_flag(out, value_col=plot_col)
|
|
2696
|
+
|
|
2697
|
+
# Format output
|
|
2698
|
+
chart_out = self._build_output_columns(
|
|
2699
|
+
df=out,
|
|
2700
|
+
value_cols=[plot_col, 'center', 'lpl', 'upl', 'beyond_limits'],
|
|
2701
|
+
)
|
|
2702
|
+
|
|
2703
|
+
chart_statistics = {
|
|
2704
|
+
'N': _mr_family_subgroup_n(mr_spec.chart_type),
|
|
2705
|
+
'center': round(center_val, spec.round_to),
|
|
2706
|
+
'lpl': round(lims['lpl'], spec.round_to),
|
|
2707
|
+
'upl': round(lims['upl'], spec.round_to),
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
# Adjust lane boundaries for chart type (no-op when offset == 0)
|
|
2711
|
+
if mr_spec.lane_boundary_offset != 0:
|
|
2712
|
+
lane_boundaries = _shift_lane_boundaries(lane_boundaries, mr_spec.lane_boundary_offset)
|
|
2713
|
+
|
|
2714
|
+
result.update(_build_ungrouped_mr_payload(
|
|
2715
|
+
mr_spec=mr_spec,
|
|
2716
|
+
chart_out=chart_out,
|
|
2717
|
+
chart_statistics=chart_statistics,
|
|
2718
|
+
plot_col=plot_col,
|
|
2719
|
+
lane_boundaries=lane_boundaries,
|
|
2720
|
+
))
|
|
2721
|
+
|
|
2722
|
+
# Optionally include intermediates for companion calculation
|
|
2723
|
+
if _return_intermediates:
|
|
2724
|
+
result['_intermediates'] = {
|
|
2725
|
+
'out': out,
|
|
2726
|
+
'mR': mR,
|
|
2727
|
+
'lane_boundaries': lane_boundaries,
|
|
2728
|
+
'is_stratified': False,
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
return result
|
|
2732
|
+
|
|
2733
|
+
def _calculate_xmr(
|
|
2734
|
+
self,
|
|
2735
|
+
value_col: str | None = None,
|
|
2736
|
+
_return_intermediates: bool = False,
|
|
2737
|
+
) -> dict:
|
|
2738
|
+
"""
|
|
2739
|
+
Calculate X (Individual) chart statistics.
|
|
2740
|
+
|
|
2741
|
+
Delegates to _calculate_mr_chart with _XMR_SPEC.
|
|
2742
|
+
|
|
2743
|
+
Parameters
|
|
2744
|
+
----------
|
|
2745
|
+
value_col : str, optional
|
|
2746
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
2747
|
+
_return_intermediates : bool, optional
|
|
2748
|
+
If True, includes '_intermediates' key for companion-mode reuse.
|
|
2749
|
+
|
|
2750
|
+
Returns
|
|
2751
|
+
-------
|
|
2752
|
+
dict
|
|
2753
|
+
Chart data: {'X': {'data': df, 'statistics': dict, 'metadata': dict}}
|
|
2754
|
+
"""
|
|
2755
|
+
return self._calculate_mr_chart(
|
|
2756
|
+
_XMR_SPEC,
|
|
2757
|
+
value_col=value_col,
|
|
2758
|
+
_return_intermediates=_return_intermediates,
|
|
2759
|
+
)
|
|
2760
|
+
|
|
2761
|
+
def _calculate_r(
|
|
2762
|
+
self,
|
|
2763
|
+
value_col: str | None = None,
|
|
2764
|
+
_precomputed: dict | None = None,
|
|
2765
|
+
) -> dict:
|
|
2766
|
+
"""
|
|
2767
|
+
Calculate mR (Moving Range) chart statistics.
|
|
2768
|
+
|
|
2769
|
+
Delegates to _calculate_mr_chart with _R_SPEC.
|
|
2770
|
+
|
|
2771
|
+
Parameters
|
|
2772
|
+
----------
|
|
2773
|
+
value_col : str, optional
|
|
2774
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
2775
|
+
_precomputed : dict, optional
|
|
2776
|
+
Pre-computed intermediates from _calculate_xmr() for companion mode.
|
|
2777
|
+
|
|
2778
|
+
Returns
|
|
2779
|
+
-------
|
|
2780
|
+
dict
|
|
2781
|
+
Chart data: {'mR': {'data': df, 'statistics': dict, 'metadata': dict}}
|
|
2782
|
+
"""
|
|
2783
|
+
return self._calculate_mr_chart(
|
|
2784
|
+
_R_SPEC,
|
|
2785
|
+
value_col=value_col,
|
|
2786
|
+
_precomputed=_precomputed,
|
|
2787
|
+
)
|
|
2788
|
+
|
|
2789
|
+
def _calculate_xmr_r(self, value_col: str | None = None) -> dict[str, ChartPayload]:
|
|
2790
|
+
"""
|
|
2791
|
+
Calculate X and mR charts together (Bishop methodology).
|
|
2792
|
+
|
|
2793
|
+
This method ensures that when companion=True, both charts are calculated
|
|
2794
|
+
efficiently using shared intermediate values. The mR chart uses the
|
|
2795
|
+
same data computed for X, avoiding redundant calculation.
|
|
2796
|
+
|
|
2797
|
+
Parameters
|
|
2798
|
+
----------
|
|
2799
|
+
value_col : str, optional
|
|
2800
|
+
Column to use for calculations. Defaults to spec.response_var.
|
|
2801
|
+
|
|
2802
|
+
Returns
|
|
2803
|
+
-------
|
|
2804
|
+
dict
|
|
2805
|
+
Combined chart data: {'X': {...}, 'mR': {...}}
|
|
2806
|
+
"""
|
|
2807
|
+
# Calculate X with intermediates for mR reuse
|
|
2808
|
+
xmr_result = self._calculate_xmr(value_col, _return_intermediates=True)
|
|
2809
|
+
|
|
2810
|
+
# Extract intermediates and remove from result
|
|
2811
|
+
intermediates = xmr_result.pop('_intermediates')
|
|
2812
|
+
|
|
2813
|
+
# Calculate mR using precomputed values
|
|
2814
|
+
r_result = self._calculate_r(value_col, _precomputed=intermediates)
|
|
2815
|
+
|
|
2816
|
+
# Combine results
|
|
2817
|
+
return {**xmr_result, **r_result}
|
|
2818
|
+
|
|
2819
|
+
def _calculate_histogram(self, value_col: str | None = None) -> dict[str, ChartPayload]:
|
|
2820
|
+
"""
|
|
2821
|
+
Calculate histogram data with mean/std statistics.
|
|
2822
|
+
|
|
2823
|
+
Parameters
|
|
2824
|
+
----------
|
|
2825
|
+
value_col : str, optional
|
|
2826
|
+
Column to use for histogram. Defaults to spec.response_var.
|
|
2827
|
+
Pass a residual column (R2, R5, etc.) for residual histograms.
|
|
2828
|
+
|
|
2829
|
+
Returns
|
|
2830
|
+
-------
|
|
2831
|
+
dict
|
|
2832
|
+
Histogram data:
|
|
2833
|
+
- For unstratified: {'Histogram': {'data': df, 'statistics': {...}, 'metadata': {...}}}
|
|
2834
|
+
- For stratified: {'Histogram': {'data': df, 'statistics': {stratum: {...}}, 'strata': [...]}}
|
|
2835
|
+
|
|
2836
|
+
Notes
|
|
2837
|
+
-----
|
|
2838
|
+
- by=[] → single histogram of all observations (default for Histogram chart)
|
|
2839
|
+
- by=['factor1'] → one histogram per factor1 level
|
|
2840
|
+
- Multi-column `by` uses tuple keys to avoid collision from string joining
|
|
2841
|
+
"""
|
|
2842
|
+
spec = self.spec
|
|
2843
|
+
if value_col is None:
|
|
2844
|
+
value_col = self.request.value_col if self.request.value_col else spec.response_var
|
|
2845
|
+
|
|
2846
|
+
data = self._source_frame().copy()
|
|
2847
|
+
values = data[value_col].dropna()
|
|
2848
|
+
|
|
2849
|
+
# Calculate global statistics
|
|
2850
|
+
n = len(values)
|
|
2851
|
+
mean = values.mean() if n > 0 else float('nan')
|
|
2852
|
+
std = values.std() if n >= 2 else float('nan')
|
|
2853
|
+
|
|
2854
|
+
# Handle stratification via `by` parameter
|
|
2855
|
+
by = list(self.request.by) if self.request.by is not None else []
|
|
2856
|
+
|
|
2857
|
+
if len(by) > 0:
|
|
2858
|
+
# Stratified histogram - one per stratum
|
|
2859
|
+
# Use pandas groupby with list of columns (no collision risk)
|
|
2860
|
+
grouped = data.groupby(by, observed=True)
|
|
2861
|
+
|
|
2862
|
+
def _py(v):
|
|
2863
|
+
"""Normalize numpy/pandas scalars to python natives."""
|
|
2864
|
+
return v.item() if hasattr(v, 'item') else v
|
|
2865
|
+
|
|
2866
|
+
# Build strata list - strings for all cases via encode_rsg
|
|
2867
|
+
if len(by) == 1:
|
|
2868
|
+
strata = data[by[0]].unique().tolist()
|
|
2869
|
+
else:
|
|
2870
|
+
strata = [encode_rsg(tuple(_py(v) for v in k)) for k in grouped.groups]
|
|
2871
|
+
|
|
2872
|
+
# Calculate per-stratum statistics
|
|
2873
|
+
per_stratum_stats = {}
|
|
2874
|
+
for stratum, group_df in grouped:
|
|
2875
|
+
# Unwrap single-element tuple for single-column groupby
|
|
2876
|
+
# groupby(['col']) returns ('value',) tuples, but strata uses scalar values
|
|
2877
|
+
key = stratum[0] if len(by) == 1 else encode_rsg(tuple(_py(v) for v in stratum))
|
|
2878
|
+
stratum_vals = group_df[value_col].dropna()
|
|
2879
|
+
stratum_n = len(stratum_vals)
|
|
2880
|
+
stratum_mean = stratum_vals.mean() if stratum_n > 0 else float('nan')
|
|
2881
|
+
stratum_std = stratum_vals.std() if stratum_n >= 2 else float('nan')
|
|
2882
|
+
_mean_rounded = round(stratum_mean, spec.round_to) if pd.notna(stratum_mean) else None
|
|
2883
|
+
_std_rounded = round(stratum_std, spec.round_to) if pd.notna(stratum_std) else None
|
|
2884
|
+
# Unified stats shape ({N, center, lpl, upl}) plus histogram
|
|
2885
|
+
# extras (mean alias, std). Histogram has no control limits.
|
|
2886
|
+
per_stratum_stats[key] = {
|
|
2887
|
+
'N': stratum_n,
|
|
2888
|
+
'center': _mean_rounded,
|
|
2889
|
+
'lpl': None,
|
|
2890
|
+
'upl': None,
|
|
2891
|
+
'mean': _mean_rounded,
|
|
2892
|
+
'std': _std_rounded,
|
|
2893
|
+
'n': stratum_n,
|
|
2894
|
+
}
|
|
2895
|
+
|
|
2896
|
+
# Keep value column and by columns for plotting
|
|
2897
|
+
output_cols = [value_col] + list(by)
|
|
2898
|
+
output_data = data[output_cols].copy()
|
|
2899
|
+
|
|
2900
|
+
# Add rsg column so focus() can filter consistently with other chart types
|
|
2901
|
+
|
|
2902
|
+
if len(by) == 1:
|
|
2903
|
+
output_data['rsg'] = encode_rsg_series(output_data, [by[0]])
|
|
2904
|
+
else:
|
|
2905
|
+
output_data['rsg'] = encode_rsg_series(output_data, list(by))
|
|
2906
|
+
|
|
2907
|
+
return {
|
|
2908
|
+
'Histogram': {
|
|
2909
|
+
'data': output_data,
|
|
2910
|
+
'statistics': per_stratum_stats,
|
|
2911
|
+
'metadata': {
|
|
2912
|
+
'chart_type': 'Histogram',
|
|
2913
|
+
'value_col': value_col,
|
|
2914
|
+
'bins': self.request.bins,
|
|
2915
|
+
'stratified': True,
|
|
2916
|
+
'stratify_col': by[0] if len(by) == 1 else '_stratify_key',
|
|
2917
|
+
'stratify_by': list(by),
|
|
2918
|
+
},
|
|
2919
|
+
'strata': strata,
|
|
2920
|
+
}
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
# Unstratified (by=[]) - full distribution
|
|
2924
|
+
output_data = data[[value_col]].copy()
|
|
2925
|
+
_mean_rounded = round(mean, spec.round_to) if pd.notna(mean) else None
|
|
2926
|
+
_std_rounded = round(std, spec.round_to) if n >= 2 and pd.notna(std) else None
|
|
2927
|
+
|
|
2928
|
+
return {
|
|
2929
|
+
'Histogram': {
|
|
2930
|
+
'data': output_data,
|
|
2931
|
+
# Unified stats shape ({N, center, lpl, upl}) plus histogram
|
|
2932
|
+
# extras (mean alias, std). Histogram has no control limits.
|
|
2933
|
+
'statistics': {
|
|
2934
|
+
'N': n,
|
|
2935
|
+
'center': _mean_rounded,
|
|
2936
|
+
'lpl': None,
|
|
2937
|
+
'upl': None,
|
|
2938
|
+
'mean': _mean_rounded,
|
|
2939
|
+
'std': _std_rounded,
|
|
2940
|
+
'n': n,
|
|
2941
|
+
},
|
|
2942
|
+
'metadata': {'chart_type': 'Histogram', 'value_col': value_col, 'bins': self.request.bins},
|
|
2943
|
+
}
|
|
2944
|
+
}
|