rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/events.py ADDED
@@ -0,0 +1,173 @@
1
+ """Heuristic associations between sequential feature alerts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from numbers import Integral
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from ._feature_values import _FeatureValueIndex
12
+ from .features import FeatureTables
13
+ from .schema import validate_rollouts
14
+ from .series import _require_columns, _validate_features
15
+
16
+ _EVENT_COLUMNS = ["event_id", "run", "onset_start", "onset_end", "n_members", "members"]
17
+ _ALERT_COLUMNS = ["run", "feature", "kind", "direction", "onset_step", "alert_step"]
18
+ _MEMBER_COLUMNS = ["event_id", *_ALERT_COLUMNS, "status"]
19
+
20
+
21
+ def _event_table(rows=()) -> pd.DataFrame:
22
+ result = pd.DataFrame(rows, columns=_EVENT_COLUMNS, dtype=object)
23
+ result["n_members"] = result.n_members.astype("int64")
24
+ return result
25
+
26
+
27
+ def _member_table(rows=()) -> pd.DataFrame:
28
+ return pd.DataFrame(rows, columns=_MEMBER_COLUMNS, dtype=object)
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class EventGrouping:
33
+ events: pd.DataFrame
34
+ members: pd.DataFrame
35
+
36
+
37
+ def group_events(
38
+ rollouts: pd.DataFrame, features: FeatureTables, alerts: pd.DataFrame
39
+ ) -> EventGrouping:
40
+ """Group rising binary alerts by deterministic first fit within each run.
41
+
42
+ Process (onset_step, alert_step, feature) order; join the first created
43
+ event whose every member matches. Pairs require numeric onset difference
44
+ <= 3 and positive-rollout Jaccard >= 0.5 over observed rollouts in the
45
+ inclusive window from the earlier onset to the later alert. Sparse
46
+ absence is zero.
47
+
48
+ Unsupported alerts, missing anchors, or no positives in an alert's own
49
+ onset-to-alert window yield singletons. Event IDs are analysis-local;
50
+ grouping does not establish a shared cause.
51
+ """
52
+ rollouts = validate_rollouts(rollouts)
53
+ _validate_features(rollouts, features)
54
+ return _group_events_prepared(_FeatureValueIndex(rollouts, features), alerts)
55
+
56
+
57
+ def _validate_alerts(values: _FeatureValueIndex, alerts: pd.DataFrame) -> None:
58
+ _require_columns(alerts, _ALERT_COLUMNS, "sequential alerts")
59
+ if alerts.duplicated(["run", "feature"]).any():
60
+ raise ValueError("duplicate sequential alert for run/feature")
61
+ for name in ["run", "feature"]:
62
+ if not alerts[name].map(lambda v: isinstance(v, str) and bool(v.strip())).all():
63
+ raise ValueError(f"alert {name} must be a nonempty string")
64
+ if not alerts.direction.isin(["up", "down"]).all():
65
+ raise ValueError("alert direction must be up or down")
66
+ if not alerts.kind.isin(["binary", "continuous"]).all():
67
+ raise ValueError("alert kind must be binary or continuous")
68
+ for name in ["onset_step", "alert_step"]:
69
+ if (
70
+ not alerts[name]
71
+ .map(
72
+ lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
73
+ )
74
+ .all()
75
+ ):
76
+ raise ValueError(f"{name} must be a nonnegative integer")
77
+ if (alerts.onset_step > alerts.alert_step).any():
78
+ raise ValueError("onset_step must not exceed alert_step")
79
+ for alert in alerts.itertuples(index=False):
80
+ if alert.run not in values.runs:
81
+ raise ValueError("alert run has no rollouts")
82
+ if alert.feature == "scalar:reward":
83
+ kind = "continuous"
84
+ elif alert.feature in values.metadata.index:
85
+ kind = values.metadata.at[alert.feature, "kind"]
86
+ else:
87
+ raise ValueError("alert feature is not in the retained catalog")
88
+ if alert.kind != kind:
89
+ raise ValueError("alert kind does not match the retained feature")
90
+
91
+
92
+ def _matches(left: dict, right: dict, steps: np.ndarray, masks: dict) -> bool:
93
+ if abs(left["onset_step"] - right["onset_step"]) > 3:
94
+ return False
95
+ start = min(left["onset_step"], right["onset_step"])
96
+ end = max(left["alert_step"], right["alert_step"])
97
+ low, high = steps.searchsorted(start), steps.searchsorted(end, side="right")
98
+ a, b = masks[left["feature"]][low:high], masks[right["feature"]][low:high]
99
+ union = np.count_nonzero(a | b)
100
+ return bool(union and 2 * np.count_nonzero(a & b) >= union)
101
+
102
+
103
+ def _group_run(values: _FeatureValueIndex, run: str, alerts: pd.DataFrame) -> list:
104
+ frame = values.runs[run]
105
+ order = frame.sort_values(["step", "sample_id"]).index.to_numpy()
106
+ steps = frame.step.to_numpy()[order]
107
+ observed = set(steps)
108
+ groups, masks = [], {}
109
+ for alert in alerts[_ALERT_COLUMNS].to_dict("records"):
110
+ status = "eligible"
111
+ if alert["kind"] != "binary":
112
+ status = "unsupported_kind"
113
+ elif alert["direction"] != "up":
114
+ status = "unsupported_direction"
115
+ elif not {alert["onset_step"], alert["alert_step"]}.issubset(observed):
116
+ status = "unavailable_window"
117
+ else:
118
+ aligned, _ = values.values(run, alert["feature"])
119
+ # Every mask uses the same validated (run, sample_id) order.
120
+ present = aligned._value.to_numpy()[order] == 1
121
+ low = steps.searchsorted(alert["onset_step"])
122
+ high = steps.searchsorted(alert["alert_step"], side="right")
123
+ if present[low:high].any():
124
+ masks[alert["feature"]] = present
125
+ else:
126
+ status = "no_positive_rollouts"
127
+ alert["status"] = status
128
+ if status == "eligible":
129
+ for members in groups:
130
+ if members[0]["status"] == "eligible" and all(
131
+ _matches(alert, member, steps, masks) for member in members
132
+ ):
133
+ members.append(alert)
134
+ break
135
+ else:
136
+ groups.append([alert])
137
+ else:
138
+ groups.append([alert])
139
+ return groups
140
+
141
+
142
+ def _group_events_prepared(
143
+ values: _FeatureValueIndex, alerts: pd.DataFrame
144
+ ) -> EventGrouping:
145
+ _validate_alerts(values, alerts)
146
+ ordered = alerts.sort_values(["run", "onset_step", "alert_step", "feature"])
147
+ events, members = [], []
148
+ for run, run_alerts in ordered.groupby("run", sort=True):
149
+ for group in _group_run(values, run, run_alerts):
150
+ event_id = f"event-{len(events) + 1:06d}"
151
+ events.append(
152
+ {
153
+ "event_id": event_id,
154
+ "run": run,
155
+ "onset_start": min(row["onset_step"] for row in group),
156
+ "onset_end": max(row["onset_step"] for row in group),
157
+ "n_members": len(group),
158
+ "members": tuple((run, row["feature"]) for row in group),
159
+ }
160
+ )
161
+ for row in group:
162
+ members.append(
163
+ {
164
+ **row,
165
+ "event_id": event_id,
166
+ "status": (
167
+ "grouped" if len(group) > 1 else "singleton_no_match"
168
+ )
169
+ if row["status"] == "eligible"
170
+ else row["status"],
171
+ }
172
+ )
173
+ return EventGrouping(_event_table(events), _member_table(members))
@@ -0,0 +1,285 @@
1
+ """Persistence and within-group reward evidence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from numbers import Real
7
+
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from ._feature_values import feature_values
12
+ from ._series_validation import validate_series
13
+ from .features import FeatureTables
14
+ from .prompt_mix import _window
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class PersistenceEvidence:
19
+ run: str
20
+ feature: str
21
+ after_step: int
22
+ reference_mean: float
23
+ change: float
24
+ effect_fraction: float
25
+ threshold: float
26
+ direction: str
27
+ status: str
28
+ n_observations: int
29
+ n_evaluable: int
30
+ n_retained: int
31
+ n_unavailable: int
32
+ fraction_retained: float | None
33
+ coverage: float | None
34
+ steps: pd.DataFrame
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class RewardAssociation:
39
+ run: str
40
+ feature: str
41
+ window: tuple[int, int]
42
+ status: str
43
+ mean_difference: float | None
44
+ n_groups: int
45
+ n_mixed_groups: int
46
+ n_rollouts: int
47
+ n_included_rollouts: int
48
+ groups: pd.DataFrame
49
+
50
+
51
+ def persistence(
52
+ series: pd.DataFrame,
53
+ *,
54
+ run: str,
55
+ feature: str,
56
+ after_step: int,
57
+ reference_mean: float,
58
+ change: float,
59
+ effect_fraction: float = 0.5,
60
+ ) -> PersistenceEvidence:
61
+ """Measure signed effect retention strictly after an observed anchor.
62
+
63
+ Retained means sign(change)*(mean-reference_mean) >= effect_fraction*abs(change).
64
+ The returned threshold is an absolute mean cutoff. Fractions use evaluable
65
+ logged observations, excluding unknown SE and reporting reduced coverage.
66
+ No tail or no evaluable rows yields no fraction.
67
+ """
68
+ frame = validate_series(series)
69
+ frame = frame.loc[frame.run.eq(run) & frame.feature.eq(feature)].copy()
70
+ return _persistence_prepared(
71
+ frame,
72
+ run=run,
73
+ feature=feature,
74
+ after_step=after_step,
75
+ reference_mean=reference_mean,
76
+ change=change,
77
+ effect_fraction=effect_fraction,
78
+ )
79
+
80
+
81
+ def _persistence_prepared(
82
+ frame: pd.DataFrame,
83
+ *,
84
+ run: str,
85
+ feature: str,
86
+ after_step: int,
87
+ reference_mean: float,
88
+ change: float,
89
+ effect_fraction: float = 0.5,
90
+ ) -> PersistenceEvidence:
91
+ """Measure persistence in one validated run/feature series."""
92
+ if type(after_step) is not int or after_step < 0:
93
+ raise ValueError("after_step must be a nonnegative integer")
94
+ for name, value in [
95
+ ("reference_mean", reference_mean),
96
+ ("change", change),
97
+ ("effect_fraction", effect_fraction),
98
+ ]:
99
+ if (
100
+ isinstance(value, bool)
101
+ or not isinstance(value, Real)
102
+ or not np.isfinite(value)
103
+ ):
104
+ raise ValueError(f"{name} must be finite and numeric")
105
+ if change == 0 or not 0 < effect_fraction <= 1:
106
+ raise ValueError("change must be nonzero and effect_fraction in (0, 1]")
107
+ if frame.empty:
108
+ raise ValueError("run/feature has no series")
109
+ if not frame.step.eq(after_step).any():
110
+ raise ValueError("after_step must identify an observed step")
111
+ binary = frame.kind.iloc[0] == "binary"
112
+ with np.errstate(over="raise", invalid="raise"):
113
+ try:
114
+ endpoint = np.float64(reference_mean) + np.float64(change)
115
+ threshold = np.float64(reference_mean) + effect_fraction * np.float64(
116
+ change
117
+ )
118
+ except FloatingPointError:
119
+ raise ValueError(
120
+ "numerical overflow computing persistence threshold"
121
+ ) from None
122
+ if binary and (
123
+ not 0 <= reference_mean <= 1
124
+ or not -4 * np.finfo(float).eps <= endpoint <= 1 + 4 * np.finfo(float).eps
125
+ ):
126
+ raise ValueError("binary reference and full effect endpoint must be in [0, 1]")
127
+ target = effect_fraction * abs(change)
128
+ if target == 0:
129
+ raise ValueError("retained effect target underflow")
130
+ tail = frame.loc[frame.step.gt(after_step)].reset_index(drop=True)
131
+ valid = tail.se.notna()
132
+ retained = pd.Series(pd.NA, index=tail.index, dtype="boolean")
133
+ sign = 1 if change > 0 else -1
134
+ try:
135
+ with np.errstate(over="raise", invalid="raise"):
136
+ departure = sign * (tail.loc[valid, "mean"].to_numpy() - reference_mean)
137
+ # Relative roundoff only: an absolute epsilon can exceed a tiny
138
+ # positive target and falsely mark an unchanged value as retained.
139
+ tolerance = 8 * np.finfo(float).eps * np.maximum(np.abs(departure), target)
140
+ retained.loc[valid] = departure >= target - tolerance
141
+ except FloatingPointError:
142
+ raise ValueError("numerical overflow computing persistence") from None
143
+ tail["retained"] = retained
144
+ n, evaluable = len(tail), int(valid.sum())
145
+ n_retained = int(retained.fillna(False).sum())
146
+ status = (
147
+ "no_future_observations"
148
+ if not n
149
+ else "unavailable_uncertainty"
150
+ if not evaluable
151
+ else "partial"
152
+ if evaluable < n
153
+ else "available"
154
+ )
155
+ return PersistenceEvidence(
156
+ run,
157
+ feature,
158
+ after_step,
159
+ float(reference_mean),
160
+ float(change),
161
+ float(effect_fraction),
162
+ float(threshold),
163
+ "up" if sign > 0 else "down",
164
+ status,
165
+ n,
166
+ evaluable,
167
+ n_retained,
168
+ n - evaluable,
169
+ n_retained / evaluable if evaluable else None,
170
+ evaluable / n if n else None,
171
+ tail,
172
+ )
173
+
174
+
175
+ _GROUP_COLUMNS = [
176
+ "run",
177
+ "step",
178
+ "prompt_id",
179
+ "n_rollouts",
180
+ "n_present",
181
+ "n_absent",
182
+ "reward_present",
183
+ "reward_absent",
184
+ "difference",
185
+ ]
186
+
187
+
188
+ def reward_association(
189
+ rollouts: pd.DataFrame,
190
+ features: FeatureTables,
191
+ *,
192
+ run: str,
193
+ feature: str,
194
+ window: tuple[int, int],
195
+ ) -> RewardAssociation:
196
+ """Average present-minus-absent reward differences over mixed prompt/step groups.
197
+
198
+ Window bounds are inclusive. Mixed groups have equal weight regardless of
199
+ completion count; pure groups remain in the detail table only.
200
+ Continuous features return ``not_binary``.
201
+ """
202
+ window = _window(window)
203
+ frame, kind = feature_values(rollouts, features, run=run, feature=feature)
204
+ return _reward_association_prepared(
205
+ frame, kind, run=run, feature=feature, window=window
206
+ )
207
+
208
+
209
+ def _reward_association_prepared(
210
+ frame: pd.DataFrame,
211
+ kind: str,
212
+ *,
213
+ run: str,
214
+ feature: str,
215
+ window: tuple[int, int],
216
+ ) -> RewardAssociation:
217
+ """Compute reward association from identity-aligned, validated values."""
218
+ window = _window(window)
219
+ frame = frame.loc[frame.step.between(*window)].copy()
220
+ if frame.empty:
221
+ raise ValueError("reward association window is empty")
222
+ keys = ["run", "step", "prompt_id"]
223
+ n_groups = len(frame[keys].drop_duplicates())
224
+ if kind != "binary":
225
+ return RewardAssociation(
226
+ run,
227
+ feature,
228
+ window,
229
+ "not_binary",
230
+ None,
231
+ n_groups,
232
+ 0,
233
+ len(frame),
234
+ 0,
235
+ pd.DataFrame(columns=_GROUP_COLUMNS),
236
+ )
237
+ present = frame["_value"].eq(1)
238
+ frame["_present_reward"] = frame.reward.where(present)
239
+ frame["_absent_reward"] = frame.reward.where(~present)
240
+ groups = (
241
+ frame.groupby(keys, sort=True)
242
+ .agg(
243
+ n_rollouts=("sample_id", "size"),
244
+ n_present=("_value", "sum"),
245
+ reward_present=("_present_reward", "mean"),
246
+ reward_absent=("_absent_reward", "mean"),
247
+ )
248
+ .reset_index()
249
+ )
250
+ groups["n_present"] = groups.n_present.astype("int64")
251
+ groups["n_absent"] = groups.n_rollouts - groups.n_present
252
+ for column, count in [
253
+ ("reward_present", "n_present"),
254
+ ("reward_absent", "n_absent"),
255
+ ]:
256
+ if not np.isfinite(groups.loc[groups[count].gt(0), column]).all():
257
+ raise ValueError("numerical overflow computing within-group reward means")
258
+ mixed = groups.n_present.gt(0) & groups.n_absent.gt(0)
259
+ groups["difference"] = np.nan
260
+ mean = None
261
+ try:
262
+ with np.errstate(over="raise", invalid="raise"):
263
+ differences = (
264
+ groups.loc[mixed, "reward_present"].to_numpy()
265
+ - groups.loc[mixed, "reward_absent"].to_numpy()
266
+ )
267
+ if len(differences):
268
+ mean = float(differences.mean())
269
+ if not np.isfinite(mean):
270
+ raise FloatingPointError
271
+ groups.loc[mixed, "difference"] = differences
272
+ except FloatingPointError:
273
+ raise ValueError("numerical overflow computing reward association") from None
274
+ return RewardAssociation(
275
+ run,
276
+ feature,
277
+ window,
278
+ "available" if mixed.any() else "no_mixed_groups",
279
+ mean,
280
+ n_groups,
281
+ int(mixed.sum()),
282
+ len(frame),
283
+ int(groups.loc[mixed, "n_rollouts"].sum()),
284
+ groups[_GROUP_COLUMNS],
285
+ )
@@ -0,0 +1,240 @@
1
+ """Rollout references for sequential alerts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from fractions import Fraction
6
+ from numbers import Integral
7
+
8
+ import pandas as pd
9
+
10
+ from ._feature_values import _FeatureValueIndex
11
+ from .features import FeatureTables
12
+ from .schema import validate_rollouts
13
+ from .series import _require_columns, _validate_features
14
+
15
+ _COLUMNS = [
16
+ "run",
17
+ "feature",
18
+ "kind",
19
+ "direction",
20
+ "onset_step",
21
+ "alert_step",
22
+ "status",
23
+ "n_window_rollouts",
24
+ "n_candidates",
25
+ "reward_selection_status",
26
+ "selection_order",
27
+ "sample_id",
28
+ "step",
29
+ "prompt_id",
30
+ "feature_value",
31
+ "reward",
32
+ "reward_group_size",
33
+ "reward_group_mean",
34
+ "reward_deviation",
35
+ "reward_deviation_status",
36
+ "selection_reason",
37
+ ]
38
+ _ALERT_COLUMNS = ["run", "feature", "kind", "direction", "onset_step", "alert_step"]
39
+ _ORDER = ["step", "run", "sample_id"]
40
+
41
+
42
+ def _exemplar_table(rows=()) -> pd.DataFrame:
43
+ result = pd.DataFrame(rows, columns=_COLUMNS)
44
+ # Null status rows must not coerce exact integer identities through float64.
45
+ for name in [
46
+ "onset_step",
47
+ "alert_step",
48
+ "step",
49
+ "selection_order",
50
+ "n_window_rollouts",
51
+ "n_candidates",
52
+ "reward_group_size",
53
+ ]:
54
+ result[name] = pd.Series([row.get(name) for row in rows], dtype=object)
55
+ for name in ["feature_value", "reward", "reward_group_mean", "reward_deviation"]:
56
+ result[name] = pd.Series([row.get(name) for row in rows], dtype=float)
57
+ return result
58
+
59
+
60
+ def select_exemplars(
61
+ rollouts: pd.DataFrame,
62
+ features: FeatureTables,
63
+ alerts: pd.DataFrame,
64
+ ) -> pd.DataFrame:
65
+ """Return distinct rollout references for rising binary sequential alerts.
66
+
67
+ From onset through alert, select up to three positives: earliest, largest
68
+ positive within-(run, step, prompt_id) reward deviation among remaining
69
+ candidates, then prompt diversity. Ties use step and canonical identity.
70
+ Response text is excluded. Unsupported/unavailable alerts yield status rows
71
+ without sample_id; no alerts yield an empty table.
72
+ """
73
+ rollouts = validate_rollouts(rollouts)
74
+ _validate_features(rollouts, features)
75
+ return _select_exemplars_prepared(_FeatureValueIndex(rollouts, features), alerts)
76
+
77
+
78
+ def _reward_deviations(frame: pd.DataFrame) -> pd.DataFrame:
79
+ """Center against all completions of the same prompt at the same step."""
80
+ if frame.empty:
81
+ raise ValueError("No objects to concatenate")
82
+ result = frame.copy()
83
+ sizes, means = [0] * len(frame), [0.0] * len(frame)
84
+ deviations, statuses = [None] * len(frame), [None] * len(frame)
85
+ reward_values = frame.reward.to_numpy()
86
+ groups = frame.groupby(["run", "step", "prompt_id"], sort=True).indices
87
+ for positions in groups.values():
88
+ rewards = [
89
+ Fraction.from_float(float(reward_values[position]))
90
+ for position in positions
91
+ ]
92
+ # Center before rounding: large offsets must not erase small deviations.
93
+ exact_mean = sum(rewards, Fraction()) / len(positions)
94
+ group_mean = float(exact_mean)
95
+ for position, reward in zip(positions, rewards, strict=True):
96
+ sizes[position], means[position] = len(positions), group_mean
97
+ try:
98
+ deviation = float(reward - exact_mean)
99
+ except OverflowError:
100
+ deviation = None
101
+ if len(positions) < 2:
102
+ statuses[position] = "insufficient_group_size"
103
+ elif deviation is None or (deviation == 0 and reward != exact_mean):
104
+ statuses[position] = "unavailable_numeric"
105
+ else:
106
+ deviations[position] = deviation
107
+ statuses[position] = "available"
108
+ result["reward_group_size"] = sizes
109
+ result["reward_group_mean"] = means
110
+ result["reward_deviation"] = pd.Series(deviations, dtype=float).to_numpy()
111
+ result["reward_deviation_status"] = statuses
112
+ return result.sort_values(_ORDER).reset_index(drop=True)
113
+
114
+
115
+ def _choose(frame: pd.DataFrame, common: dict) -> list[dict]:
116
+ candidates = frame.loc[frame._value.eq(1)].reset_index(drop=True)
117
+ common["n_candidates"] = len(candidates)
118
+ if candidates.empty:
119
+ return [
120
+ {
121
+ **common,
122
+ "status": "no_positive_rollouts",
123
+ "reward_selection_status": "no_candidates",
124
+ }
125
+ ]
126
+
127
+ chosen = [(0, "earliest_occurrence")]
128
+ remaining = candidates.iloc[1:]
129
+ comparable = remaining.loc[remaining.reward_deviation_status.eq("available")]
130
+ positive = comparable.loc[comparable.reward_deviation.gt(0)]
131
+ if remaining.empty:
132
+ reward_status = "no_remaining_candidates"
133
+ elif comparable.empty:
134
+ reward_status = "no_comparable_candidates"
135
+ elif positive.empty:
136
+ reward_status = "no_positive_deviation"
137
+ else:
138
+ reward_status = "selected"
139
+ index = positive.sort_values(
140
+ ["reward_deviation", *_ORDER], ascending=[False, True, True, True]
141
+ ).index[0]
142
+ chosen.append((index, "high_within_prompt_reward_deviation"))
143
+
144
+ while len(chosen) < min(3, len(candidates)):
145
+ indices = [index for index, _ in chosen]
146
+ remaining = candidates.drop(index=indices)
147
+ new_prompts = remaining.loc[
148
+ ~remaining.prompt_id.isin(candidates.loc[indices, "prompt_id"])
149
+ ]
150
+ if new_prompts.empty:
151
+ chosen.append((remaining.index[0], "earliest_remaining"))
152
+ else:
153
+ chosen.append((new_prompts.index[0], "prompt_diversity"))
154
+
155
+ rows = []
156
+ for order, (index, reason) in enumerate(chosen, start=1):
157
+ candidate = candidates.loc[index]
158
+ rows.append(
159
+ {
160
+ **common,
161
+ "status": "selected",
162
+ "selection_order": order,
163
+ "reward_selection_status": reward_status,
164
+ **{
165
+ name: candidate[name]
166
+ for name in [
167
+ "sample_id",
168
+ "step",
169
+ "prompt_id",
170
+ "reward",
171
+ "reward_group_size",
172
+ "reward_group_mean",
173
+ "reward_deviation",
174
+ "reward_deviation_status",
175
+ ]
176
+ },
177
+ "feature_value": candidate["_value"],
178
+ "selection_reason": reason,
179
+ }
180
+ )
181
+ return rows
182
+
183
+
184
+ def _select_exemplars_prepared(
185
+ values: _FeatureValueIndex, alerts: pd.DataFrame
186
+ ) -> pd.DataFrame:
187
+ _require_columns(alerts, _ALERT_COLUMNS, "sequential alerts")
188
+ if alerts.duplicated(["run", "feature"]).any():
189
+ raise ValueError("duplicate sequential alert for run/feature")
190
+ for name in ["run", "feature"]:
191
+ if not alerts[name].map(lambda v: isinstance(v, str) and bool(v.strip())).all():
192
+ raise ValueError(f"alert {name} must be a nonempty string")
193
+ if not alerts.direction.isin(["up", "down"]).all():
194
+ raise ValueError("alert direction must be up or down")
195
+ if not alerts.kind.isin(["binary", "continuous"]).all():
196
+ raise ValueError("alert kind must be binary or continuous")
197
+ for name in ["onset_step", "alert_step"]:
198
+ if (
199
+ not alerts[name]
200
+ .map(
201
+ lambda v: isinstance(v, Integral) and not isinstance(v, bool) and v >= 0
202
+ )
203
+ .all()
204
+ ):
205
+ raise ValueError(f"{name} must be a nonnegative integer")
206
+ if (alerts.onset_step > alerts.alert_step).any():
207
+ raise ValueError("onset_step must not exceed alert_step")
208
+
209
+ rows = []
210
+ for alert in alerts.sort_values(["run", "feature"]).itertuples(index=False):
211
+ if alert.run not in values.runs:
212
+ raise ValueError("alert run has no rollouts")
213
+ if (
214
+ alert.feature != "scalar:reward"
215
+ and alert.feature not in values.metadata.index
216
+ ):
217
+ raise ValueError("alert feature is not in the retained catalog")
218
+ frame, kind = values.values(alert.run, alert.feature)
219
+ if alert.kind != kind:
220
+ raise ValueError("alert kind does not match the retained feature")
221
+ window = frame.loc[frame.step.between(alert.onset_step, alert.alert_step)]
222
+ common = {
223
+ "run": alert.run,
224
+ "feature": alert.feature,
225
+ "kind": kind,
226
+ "direction": alert.direction,
227
+ "onset_step": int(alert.onset_step),
228
+ "alert_step": int(alert.alert_step),
229
+ "n_window_rollouts": len(window),
230
+ }
231
+ if kind != "binary":
232
+ rows.append({**common, "status": "unsupported_kind"})
233
+ elif alert.direction != "up":
234
+ rows.append({**common, "status": "unsupported_direction"})
235
+ elif not {alert.onset_step, alert.alert_step}.issubset(set(window.step)):
236
+ rows.append({**common, "status": "unavailable_window"})
237
+ else:
238
+ ordered = window.sort_values(_ORDER).reset_index(drop=True)
239
+ rows.extend(_choose(_reward_deviations(ordered), common))
240
+ return _exemplar_table(rows)